diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..9e01cfd --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,22 @@ +name: Publish to pypi +on: [push] +jobs: + Publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 2.x + - name: Install dependencies + run: >- + python -m pip install --user --upgrade setuptools wheel + - name: Build + run: >- + python setup.py sdist bdist_wheel + - name: Publish package + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index f7ac792..f860931 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,22 @@ .coverage .noseids +.build +*.pyc + +\#*\# +.idea +gaepdb.py +*.xml + +# for sphinx documentation +doc/build/* +doc/source/* +!doc/source/conf.py +!doc/source/index.txt +!doc/source/design*.txt +!doc/source/_static/* +!doc/source/_templates/* + +bilobac +dist/ +furious.egg-info/ diff --git a/.travis.yml b/.travis.yml index 9e9ec71..961cd11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,27 @@ +sudo: false language: python python: - - "2.7" + - "2.7" -# Setup the GAE SDK. -before_script: - - wget http://googleappengine.googlecode.com/files/google_appengine_1.7.4.zip -nv - - unzip -q google_appengine_1.7.4.zip +# Cache our Gcloud SDK between commands +cache: + directories: + - "$HOME/google-cloud-sdk/" -# command to install dependencies -install: "pip install -r requirements.txt --use-mirrors" +env: +# Make sure gcloud command is on our PATH and the App Engine SDK is in the Python path +- GAE_PYTHONPATH=${HOME}/.cache/google_appengine PATH=$PATH:${HOME}/google-cloud-sdk/bin PYTHONPATH=${PYTHONPATH}:${GAE_PYTHONPATH} CLOUDSDK_CORE_DISABLE_PROMPTS=1 -# command to run tests -script: nosetests --gae-lib-path=./google_appengine +before_install: +# Install Google App Engine Python SDK +- if [ ! -d "${GAE_PYTHONPATH}" ]; then + python scripts/fetch_gae_sdk.py $(dirname "${GAE_PYTHONPATH}"); + fi +install: +- pip install -r requirements_dev.txt + +script: + - nosetests --gae-lib-path=./google_appengine --with-coverage + - python setup.py sdist + - pip install . diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1da4c37 --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +SHELL := /bin/bash +PYTHON := python +PIP := pip + +BUILD_DIR := .build + +clean: + find . -name "*.py[co]" -delete + rm -f .coverage + +distclean: clean + rm -rf $(BUILD_DIR) + +run: deps + dev_appserver.py . + +deps: py_dev_deps + +py_dev_deps: $(BUILD_DIR)/pip-dev.log + +$(BUILD_DIR)/pip-dev.log: requirements_dev.txt + @mkdir -p .build + $(PIP) install -Ur requirements_dev.txt | tee $(BUILD_DIR)/pip-dev.log + +unit: clean + nosetests + +integrations: + nosetests --logging-level=ERROR -a slow --with-coverage + +test: clean integrations + diff --git a/README.md b/README.md index 7ee93b6..b917604 100644 --- a/README.md +++ b/README.md @@ -21,20 +21,34 @@ Usage In the simplest form, usage looks like: - from furious import Async + from furious.async import Async # Create an Async object. async = Async( target="your.module.func", - args=("positstional", "args"), - kwargs={"kwargs": "too"}) + args=("positional", "args"), + kwargs={"foo": "bar"}) # Tell the async to insert itself to be run. async.start() This inserts a task that will make the following call: - your.module.func("pos", "args", kwarg="too") + your.module.func("positional", "args", foo="bar") + +**CAUTION**: Arguments **must** be json-serializable. +Contrast that with the [deferred lib](https://cloud.google.com/appengine/articles/deferred), +which allows any picklable type +(see [What can be pickled and unpickled?](https://docs.python.org/2/library/pickle.html#what-can-be-pickled-and-unpickled)). + +Why did furious choose this limitation? +When a python object that is pickled gets moved in the code base, +attempts to unpickle it will blow up. Furious tasks can run on +different instances at a future time by which there may have +been a code deploy done. References: + +* [Alex Gaynor: Pickles are for Delis, not Software - PyCon 2014 - YouTube](https://www.youtube.com/watch?v=7KnfGDajDQw&t=1292) +* [Don't Pickle Your Data](http://www.benfrederickson.com/dont-pickle-your-data/) ### Grouping async jobs @@ -56,7 +70,8 @@ You can group jobs together, It is possible to set options, like the target queue, - from furious import context, defaults + from furious import context + from furious.async import defaults @defaults(queue='square') def square_a_number(number): @@ -89,7 +104,7 @@ Tasks targeted at the same queue will be batch inserted. Contexts allow you to build workflows easily, - from furious import Async + from furious.async import Async from furious.context import get_current_context, new def square_a_number(number): @@ -121,7 +136,7 @@ Contexts allow you to build workflows easily, Asyncs and Contexts maybe nested to build more complex workflows. - from furious import Async + from furious.async import Async from furious.context import new def do_some_work(number): diff --git a/aviary.yaml b/aviary.yaml new file mode 100644 index 0000000..3c59652 --- /dev/null +++ b/aviary.yaml @@ -0,0 +1,15 @@ +version: 1 + +exclude: + - tests?/ + +raven_monitored_classes: null + +raven_monitored_files: + ^furious/extras/xsrf.py$: + reason: CSRF protection helper methods + +raven_monitored_functions: null + +raven_monitored_keywords: null + diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..ba099bc --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/furious.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/furious.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/furious" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/furious" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/how_to_build_docs.txt b/doc/how_to_build_docs.txt new file mode 100644 index 0000000..9bd63b5 --- /dev/null +++ b/doc/how_to_build_docs.txt @@ -0,0 +1,33 @@ +1. You must have a .pth file in your site-packages directory with +the path of your google_appengine sdk. For Unix environments just create a file here: + +$VIRTUAL_ENV/lib/python2.7/site-packages/appengine.pth + +with the path: + +/usr/local/google_appengine + + +2. Install the requirements: + + $ pip install -r doc/requirements.txt + + +3. Auto build doc source from project source (run from the doc directory): +(http://sphinx-doc.org/invocation.html#invocation-apidoc) + $ sphinx-apidoc -T -s txt -o source ../furious + $ sphinx-apidoc -T -s txt -o source ../example + +to force rebuild the rsts, include -f: +(http://sphinx-doc.org/invocation.html) + $ sphinx-apidoc -T -f -s txt -o source ../furious + $ sphinx-apidoc -T -f -s txt -o source ../example + + +4. Build docs from source: + $ make html + +Sometimes the docs don't update and you need to remove doc/build/doctrees: + $ rm -Rf build/doctrees + + diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 0000000..6716b51 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,190 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +set I18NSPHINXOPTS=%SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\furious.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\furious.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..db20d00 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,6 @@ +Jinja2==2.6 +Pygments==1.6 +Sphinx==1.1.3 +docutils==0.12 +python-termstyle==0.1.10 +wsgiref==0.1.2 diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..8daf856 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,277 @@ +# -*- coding: utf-8 -*- +# +# furious documentation build configuration file, created by +# sphinx-quickstart on Wed Mar 6 11:23:29 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath('.')))) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath('.'))), 'lib')) + +import dev_appserver + +sys.path.extend(dev_appserver.EXTRA_PATHS) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.pngmath', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode'] + + +# If true, the todo will be printed in the documentation +todo_include_todos = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.txt' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'furious' +copyright = u'2013, WebFilings' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.5' +# The full version, including alpha/beta/rc tags. +release = '1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'furiousdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'furious.tex', u'furious Documentation', + u'WebFilings', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'furious', u'Furious Documentation', + [u'WebFilings'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'furious', u'Furious Documentation', + u'WebFilings, LLC', 'furious', 'Asynchronously execute lots of tasks.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'http://docs.python.org/': None} + +# autodoc + + +def skip(app, what, name, obj, skip, options): + if name == "__init__": + return False + return skip + + +def setup(app): + app.connect("autodoc-skip-member", skip) diff --git a/doc/source/index.txt b/doc/source/index.txt new file mode 100644 index 0000000..da79337 --- /dev/null +++ b/doc/source/index.txt @@ -0,0 +1,9 @@ +Furious Project +=============== + +.. toctree:: + :maxdepth: 7 + + example + furious + design diff --git a/example/__init__.py b/example/__init__.py index ac5296d..25a0d77 100644 --- a/example/__init__.py +++ b/example/__init__.py @@ -21,259 +21,50 @@ more complicated processing pipelines. """ -import logging - import webapp2 -from furious.async import defaults - - -class AsyncIntroHandler(webapp2.RequestHandler): - """Demonstrate the creation and insertion of a single furious task.""" - def get(self): - from furious.async import Async - - # Instantiate an Async object. - async_task = Async( - target=example_function, args=[1], kwargs={'some': 'value'}) - - # Insert the task to run the Async object, note that it may begin - # executing immediately or with some delay. - async_task.start() - - logging.info('Async job kicked off.') - - self.response.out.write('Successfully inserted Async job.') - - -class ContextIntroHandler(webapp2.RequestHandler): - """Demonstrate using a Context to batch insert a group of furious tasks.""" - def get(self): - from furious.async import Async - from furious import context - - # Create a new furious Context. - with context.new() as ctx: - # "Manually" instantiate and add an Async object to the Context. - async_task = Async( - target=example_function, kwargs={'first': 'async'}) - ctx.add(async_task) - logging.info('Added manual job to context.') - - # Use the shorthand style, note that add returns the Async object. - for i in xrange(5): - ctx.add(target=example_function, args=[i]) - logging.info('Added job %d to context.', i) - - # When the Context is exited, the tasks are inserted (if there are no - # errors). - - logging.info('Async jobs for context batch inserted.') - - self.response.out.write('Successfully inserted a group of Async jobs.') - - -class AsyncCallbackHandler(webapp2.RequestHandler): - """Demonstrate setting an Async callback.""" - def get(self): - from furious.async import Async - - # Instantiate an Async object, specifying a 'success' callback. - async_task = Async( - target=example_function, args=[1], kwargs={'some': 'value'}, - callbacks={'success': all_done} - ) - - # Insert the task to run the Async object. The success callback will - # be executed in the furious task after the job is executed. - async_task.start() - - logging.info('Async job kicked off.') - - self.response.out.write('Successfully inserted Async job.') - - -class AsyncErrorCallbackHandler(webapp2.RequestHandler): - """Demonstrate handling an error using an Async callback.""" - def get(self): - from furious.async import Async - - # Instantiate an Async object, specifying a 'error' callback. - async_task = Async( - target=dir, args=[1, 2, 3], - callbacks={'error': handle_an_error} - ) - - # Insert the task to run the Async object. The error callback will be - # executed in the furious task after the job has raised an exception. - async_task.start() - - logging.info('Erroneous Async job kicked off.') - - self.response.out.write('Successfully inserted Async job.') - - -class AsyncAsyncCallbackHandler(webapp2.RequestHandler): - """Demonstrate using an Async as a callback for another Async.""" - def get(self): - from furious.async import Async - - # Instantiate an Async object to act as our success callback. - # NOTE: Your async.result is not directly available from the - # success_callback Async, you will need to persist the result - # and fetch it from the other Async if needed. - success_callback = Async( - target=example_function, kwargs={'it': 'worked'} - ) - - # Instantiate an Async object, setting the success_callback to the - # above Async object. - async_task = Async( - target=example_function, kwargs={'trigger': 'job'}, - callbacks={'success': success_callback} - ) - - # Insert the task to run the Async object. - async_task.start() - - logging.info('Async job kicked off.') - - self.response.out.write('Successfully inserted Async job.') - - -class SimpleWorkflowHandler(webapp2.RequestHandler): - """Demonstrate constructing a simple state machine.""" - def get(self): - from furious.async import Async - - # Instantiate an Async object to start the chain. - Async(target=simple_state_machine).start() - - logging.info('Async chain kicked off.') - - self.response.out.write('Successfully inserted Async chain starter.') - - -class ComplexWorkflowHandler(webapp2.RequestHandler): - """Demonstrate constructing a more complex state machine.""" - def get(self): - from furious.async import Async - - # Instantiate an Async object to start the machine in state alpha. - Async(target=complex_state_generator_alpha).start() - - logging.info('Async chain kicked off.') - - self.response.out.write('Successfully inserted Async chain starter.') - - -def example_function(*args, **kwargs): - """This function is called by furious tasks to demonstrate usage.""" - logging.info('example_function executed with args: %r, kwargs: %r', - args, kwargs) - - return args - - -def all_done(): - """Will be run if the async task runs successfully.""" - from furious.context import get_current_async - - async = get_current_async() - - logging.info('async task complete, value returned: %r', async.result) - - -def handle_an_error(): - """Will be run if the async task raises an unhandled exception.""" - import os - - from furious.context import get_current_async - - exception_info = get_current_async().result - - logging.info('async job blew up, exception info: %r', exception_info) - - retries = int(os.environ['HTTP_X_APPENGINE_TASKRETRYCOUNT']) - if retries < 2: - raise exception_info.exception - else: - logging.info('Caught too many errors, giving up now.') - - -def simple_state_machine(): - """Pick a number, if it is more than some cuttoff continue the chain.""" - from random import random - - from furious.async import Async - - number = random() - logging.info('Generating a number... %s', number) - - if number > 0.25: - logging.info('Continuing to do stuff.') - return Async(target=simple_state_machine) - - return number - - -@defaults(callbacks={'success': "example.state_machine_success"}) -def complex_state_generator_alpha(last_state=''): - """Pick a state.""" - from random import choice - - states = ['ALPHA', 'ALPHA', 'ALPHA', 'BRAVO', 'BRAVO', 'DONE'] - if last_state: - states.remove(last_state) # Slightly lower chances of previous state. - - state = choice(states) - - logging.info('Generating a state... %s', state) - - return state - - -@defaults(callbacks={'success': "example.state_machine_success"}) -def complex_state_generator_bravo(last_state=''): - """Pick a state.""" - from random import choice - - states = ['ALPHA', 'BRAVO', 'BRAVO', 'DONE'] - if last_state: - states.remove(last_state) # Slightly lower chances of previous state. - - state = choice(states) - - logging.info('Generating a state... %s', state) - - return state - - -def state_machine_success(): - """A positive result! Iterate!""" - from furious.async import Async - from furious.context import get_current_async - - result = get_current_async().result - - if result == 'ALPHA': - logging.info('Inserting continuation for state %s.', result) - return Async(target=complex_state_generator_alpha, args=[result]) - - elif result == 'BRAVO': - logging.info('Inserting continuation for state %s.', result) - return Async(target=complex_state_generator_bravo, args=[result]) - - logging.info('Done working, stop now.') +from .abort_and_restart import AbortAndRestartHandler +from .async_intro import AsyncIntroHandler +from .batcher import BatcherHandler +from .batcher import BatcherStatsHandler +from .batcher import BatcherViewHandler +from .callback import AsyncCallbackHandler +from .callback import AsyncErrorCallbackHandler +from .callback import AsyncAsyncCallbackHandler +from .complex_workflow import ComplexWorkflowHandler +from .context_intro import ContextIntroHandler +from .context_events import ContextEventsHandler +from .context_inherit import ContextInheritHandler +from .context_completion_with_results import ContextCompletionHandler +from .grep import GrepHandler +from .simple_workflow import SimpleWorkflowHandler +from .limits import LimitHandler + +from furious.handlers.webapp import AsyncJobHandler + +config = { + 'webapp2_extras.jinja2': { + 'template_path': 'example/templates' + } +} app = webapp2.WSGIApplication([ + ('/_queue/async.*', AsyncJobHandler), ('/', AsyncIntroHandler), + ('/abort_and_restart', AbortAndRestartHandler), ('/context', ContextIntroHandler), + ('/context/completion', ContextCompletionHandler), + ('/context/event', ContextEventsHandler), + ('/context/inherit', ContextInheritHandler), ('/callback', AsyncCallbackHandler), ('/callback/error', AsyncErrorCallbackHandler), ('/callback/async', AsyncAsyncCallbackHandler), ('/workflow', SimpleWorkflowHandler), ('/workflow/complex', ComplexWorkflowHandler), -]) - + ('/batcher', BatcherViewHandler), + ('/batcher/run', BatcherHandler), + ('/batcher/stats', BatcherStatsHandler), + ('/grep', GrepHandler), + ('/limits', LimitHandler), +], config=config) diff --git a/example/abort_and_restart.py b/example/abort_and_restart.py new file mode 100644 index 0000000..57b8311 --- /dev/null +++ b/example/abort_and_restart.py @@ -0,0 +1,61 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A basic example to demonstrate the AbortAndRestart functionality.""" + +import logging + +import webapp2 + + +class AbortAndRestartHandler(webapp2.RequestHandler): + """Demonstrate the AbortAndRestart functionality.""" + + def get(self): + from furious.async import Async + + # Instantiate an Async + async_task = Async(target=aborting_function) + + # Start an Async() + async_task.start() + + logging.info('Original Async kicked off.') + + self.response.write('Successfully inserted AbortAndRestart example.') + + +def aborting_function(): + """There is a 50% chance that this function will AbortAndRestart or + complete successfully. + + The 50% chance simply represents a process that will fail half the time + and succeed half the time. + """ + import random + + logging.info('In aborting_function') + + if random.random() < .5: + from furious.errors import AbortAndRestart + + logging.info('Getting ready to restart') + + # Raise AbortAndRestart like an Exception, and watch the magic happen. + raise AbortAndRestart() + + logging.info('No longer restarting') + diff --git a/example/async_intro.py b/example/async_intro.py new file mode 100644 index 0000000..a48a737 --- /dev/null +++ b/example/async_intro.py @@ -0,0 +1,50 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A very basic async example. + +This will insert an async task to run the example_function. +""" + +import logging + +import webapp2 + + +class AsyncIntroHandler(webapp2.RequestHandler): + """Demonstrate the creation and insertion of a single furious task.""" + def get(self): + from furious.async import Async + + # Instantiate an Async object. + async_task = Async( + target=example_function, args=[1], kwargs={'some': 'value'}) + + # Insert the task to run the Async object, note that it may begin + # executing immediately or with some delay. + async_task.start() + + logging.info('Async job kicked off.') + + self.response.out.write('Successfully inserted Async job.') + + +def example_function(*args, **kwargs): + """This function is called by furious tasks to demonstrate usage.""" + logging.info('example_function executed with args: %r, kwargs: %r', + args, kwargs) + + return args diff --git a/example/batcher/__init__.py b/example/batcher/__init__.py new file mode 100644 index 0000000..c4f3858 --- /dev/null +++ b/example/batcher/__init__.py @@ -0,0 +1,244 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import datetime +import time +import json +import logging + +import webapp2 + +from webapp2_extras import jinja2 + + +class BatcherViewHandler(webapp2.RequestHandler): + """Batcher view to insert tasks and view the stats. + + url: host/batcher + """ + + @webapp2.cached_property + def jinja2(self): + # Returns a Jinja2 renderer cached in the app registry. + return jinja2.get_jinja2(app=self.app) + + def render_response(self, _template, **context): + # Renders a template and writes the result to the response. + rv = self.jinja2.render_template(_template, **context) + self.response.write(rv) + + def get(self): + context = {} + + self.render_response('batcher.html', **context) + + +class BatcherHandler(webapp2.RequestHandler): + """Handler for inserting batch messages. Takes urlparams of color, value + and count. The color is just a tag and value is an incrementing number for + that tag. The count is the number of messages with that payload to insert. + """ + + def get_params(self): + color = self.request.GET['color'] + value = self.request.GET['value'] + count = int(self.request.GET['count']) + + assert color + assert value + assert count + + return color.strip(), value.strip(), count + + def get(self): + from furious import context + from furious.batcher import Message + from furious.batcher import MessageProcessor + + try: + color, value, count = self.get_params() + except (KeyError, AssertionError): + response = { + "success": False, + "message": "Invalid parameters." + } + self.response.write(json.dumps(response)) + return + + payload = { + "color": color, + "value": value, + "timestamp": time.mktime(datetime.datetime.utcnow().timetuple()) + } + + tag = "color" + + # create a context to insert multiple Messages + with context.new() as ctx: + # loop through the count adding a task to the context per increment + for _ in xrange(count): + # insert the message with the payload + ctx.add(Message(task_args={"payload": payload, "tag": tag})) + + # insert a processor to fetch the messages in batches + # this should always be inserted. the logic will keep it from inserting + # too many processors + processor = MessageProcessor( + target=process_messages, args=(tag,), tag=tag, + task_args={"countdown": 0}) + processor.start() + + response = { + "success": True, + "message": "Task inserted successfully with %s" % (payload,) + } + + self.response.write(json.dumps(response)) + + +class BatcherStatsHandler(webapp2.RequestHandler): + """Handler for returing the stats to the client. Returns as a json payload. + Pulls the stats from memcache. + """ + + def get(self): + from google.appengine.api import memcache + + stats = memcache.get('color') + stats = stats if stats else json.dumps(get_default_stats()) + self.response.write(stats) + + +def process_messages(tag, retries=0): + """Processes the messages pulled fromm a queue based off the tag passed in. + Will insert another processor if any work was processed or the retry count + is under the max retry count. Will update a aggregated stats object with + the data in the payload of the messages processed. + + :param tag: :class: `str` Tag to query the queue on + :param retry: :class: `int` Number of retries the job has processed + """ + from furious.batcher import bump_batch + from furious.batcher import MESSAGE_DEFAULT_QUEUE + from furious.batcher import MessageIterator + from furious.batcher import MessageProcessor + + from google.appengine.api import memcache + + # since we don't have a flag for checking complete we'll re-insert a + # processor task with a retry count to catch any work that may still be + # filtering in. If we've hit our max retry count we just bail out and + # consider the job complete. + if retries > 5: + logging.info("Process messages hit max retry and is exiting") + return + + # create a message iteragor for the tag in batches of 500 + message_iterator = MessageIterator(tag, MESSAGE_DEFAULT_QUEUE, 500) + + client = memcache.Client() + + # get the stats object from cache + stats = client.gets(tag) + + # json decode it if it exists otherwise get the default state. + stats = json.loads(stats) if stats else get_default_stats() + + work_processed = False + + # loop through the messages pulled from the queue. + for message in message_iterator: + work_processed = True + + value = int(message.get("value", 0)) + color = message.get("color").lower() + + # update the total stats with the value pulled + set_stats(stats["totals"], value) + + # update the specific color status via the value pulled + set_stats(stats["colors"][color], value) + + # insert the stats back into cache + json_stats = json.dumps(stats) + + # try and do an add first to see if it's new. We can't trush get due to + # a race condition. + if not client.add(tag, json_stats): + # if we couldn't add than lets do a compare and set to safely + # update the stats + if not client.cas(tag, json_stats): + raise Exception("Transaction Collision.") + + # bump the process batch id + bump_batch(tag) + + if work_processed: + # reset the retries as we've processed work + retries = 0 + else: + # no work was processed so increment the retries + retries += 1 + + # insert another processor + processor = MessageProcessor( + target=process_messages, args=("colors",), + kwargs={'retries': retries}, tag="colors") + + processor.start() + + +def set_stats(stats, value): + """Updates the stats with the value passed in. + + :param stats: :class: `dict` + :param value: :class: `int` + """ + stats["total_count"] += 1 + stats["value"] += value + stats["average"] = stats["value"] / stats["total_count"] + + # this is just a basic example and not the best way to track aggregation. + # for max and min old there are cases where this will not work correctly. + if value > stats["max"]: + stats["max"] = value + + if value < stats["min"] or stats["min"] == 0: + stats["min"] = value + + +def get_default_stats(): + """Returns a :class: `dict` of the default stats structure.""" + + default_stats = { + "total_count": 0, + "max": 0, + "min": 0, + "value": 0, + "average": 0, + "last_update": None, + } + + return { + "totals": default_stats, + "colors": { + "red": default_stats.copy(), + "blue": default_stats.copy(), + "yellow": default_stats.copy(), + "green": default_stats.copy(), + "black": default_stats.copy(), + } + } diff --git a/example/callback.py b/example/callback.py new file mode 100644 index 0000000..55f4eeb --- /dev/null +++ b/example/callback.py @@ -0,0 +1,140 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Async Callback Examples. + +There are 3 examples below. + +1. AsyncCallbackHandler +Registering a function as a callback to be triggered when the job has +completed. + +2. AsyncErrorCallbackHandler +Registering a function as an error callback to be triggered when an error has +been hit in the Async process. + +3. AsyncAsyncCallbackHandler +Registering another Async object as a callback to be inserted when the job has +complated. +""" + + +import logging + +import webapp2 + + +class AsyncCallbackHandler(webapp2.RequestHandler): + """Demonstrate setting an Async callback.""" + def get(self): + from furious.async import Async + + # Instantiate an Async object, specifying a 'success' callback. + async_task = Async( + target=example_function, args=[1], kwargs={'some': 'value'}, + callbacks={'success': all_done} + ) + + # Insert the task to run the Async object. The success callback will + # be executed in the furious task after the job is executed. + async_task.start() + + logging.info('Async job kicked off.') + + self.response.out.write('Successfully inserted Async job.') + + +class AsyncErrorCallbackHandler(webapp2.RequestHandler): + """Demonstrate handling an error using an Async callback.""" + def get(self): + from furious.async import Async + + # Instantiate an Async object, specifying a 'error' callback. + async_task = Async( + target=dir, args=[1, 2, 3], + callbacks={'error': handle_an_error} + ) + + # Insert the task to run the Async object. The error callback will be + # executed in the furious task after the job has raised an exception. + async_task.start() + + logging.info('Erroneous Async job kicked off.') + + self.response.out.write('Successfully inserted Async job.') + + +class AsyncAsyncCallbackHandler(webapp2.RequestHandler): + """Demonstrate using an Async as a callback for another Async.""" + def get(self): + from furious.async import Async + + # Instantiate an Async object to act as our success callback. + # NOTE: Your async.result is not directly available from the + # success_callback Async, you will need to persist the result + # and fetch it from the other Async if needed. + success_callback = Async( + target=example_function, kwargs={'it': 'worked'} + ) + + # Instantiate an Async object, setting the success_callback to the + # above Async object. + async_task = Async( + target=example_function, kwargs={'trigger': 'job'}, + callbacks={'success': success_callback} + ) + + # Insert the task to run the Async object. + async_task.start() + + logging.info('Async job kicked off.') + + self.response.out.write('Successfully inserted Async job.') + + +def example_function(*args, **kwargs): + """This function is called by furious tasks to demonstrate usage.""" + logging.info('example_function executed with args: %r, kwargs: %r', + args, kwargs) + + return args + + +def all_done(): + """Will be run if the async task runs successfully.""" + from furious.context import get_current_async + + async = get_current_async() + + logging.info('async task complete, value returned: %r', async.result) + + +def handle_an_error(): + """Will be run if the async task raises an unhandled exception.""" + import os + + from furious.context import get_current_async + + async = get_current_async() + async_exception = async.result.payload + exc_info = async_exception.traceback + logging.info('async job blew up, exception info: %r', exc_info) + + retries = int(os.environ['HTTP_X_APPENGINE_TASKRETRYCOUNT']) + if retries < 2: + raise Exception(async_exception.error) + else: + logging.info('Caught too many errors, giving up now.') diff --git a/example/complex_workflow.py b/example/complex_workflow.py new file mode 100644 index 0000000..281e356 --- /dev/null +++ b/example/complex_workflow.py @@ -0,0 +1,91 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A complex workflow example. + +This example uses a couple different functions to chain together in a worklow. +""" + +import logging + +import webapp2 + +from furious.async import defaults + + +class ComplexWorkflowHandler(webapp2.RequestHandler): + """Demonstrate constructing a more complex state machine.""" + def get(self): + from furious.async import Async + + # Instantiate an Async object to start the machine in state alpha. + Async(target=complex_state_generator_alpha).start() + + logging.info('Async chain kicked off.') + + self.response.out.write('Successfully inserted Async chain starter.') + + +@defaults( + callbacks={'success': "example.complex_workflow.state_machine_success"}) +def complex_state_generator_alpha(last_state=''): + """Pick a state.""" + from random import choice + + states = ['ALPHA', 'ALPHA', 'ALPHA', 'BRAVO', 'BRAVO', 'DONE'] + if last_state: + states.remove(last_state) # Slightly lower chances of previous state. + + state = choice(states) + + logging.info('Generating a state... %s', state) + + return state + + +@defaults( + callbacks={'success': "example.complex_workflow.state_machine_success"}) +def complex_state_generator_bravo(last_state=''): + """Pick a state.""" + from random import choice + + states = ['ALPHA', 'BRAVO', 'BRAVO', 'DONE'] + if last_state: + states.remove(last_state) # Slightly lower chances of previous state. + + state = choice(states) + + logging.info('Generating a state... %s', state) + + return state + + +def state_machine_success(): + """A positive result! Iterate!""" + from furious.async import Async + from furious.context import get_current_async + + result = get_current_async().result + + if result == 'ALPHA': + logging.info('Inserting continuation for state %s.', result) + return Async(target=complex_state_generator_alpha, args=[result]) + + elif result == 'BRAVO': + logging.info('Inserting continuation for state %s.', result) + return Async(target=complex_state_generator_bravo, args=[result]) + + logging.info('Done working, stop now.') diff --git a/example/context_completion_with_results.py b/example/context_completion_with_results.py new file mode 100644 index 0000000..708c504 --- /dev/null +++ b/example/context_completion_with_results.py @@ -0,0 +1,79 @@ +# +# Copyright 2014 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Example of using Context events for work flow. + +This example creates a context with events, then adds a few Async jobs to the +Context. +""" + + +import logging + +import webapp2 + + +class ContextCompletionHandler(webapp2.RequestHandler): + """Demonstrate using Context Events to make work flows.""" + def get(self): + from furious.async import Async + from furious import context + + count = self.request.get('tasks', 5) + + # Create a new furious Context with persistance enabled. + with context.new(persist_async_results=True) as ctx: + # Set a completion event handler. + ctx.set_event_handler('complete', + Async(context_complete, args=[ctx.id])) + + # Insert some Asyncs. + for i in xrange(int(count)): + ctx.add(target=async_worker, args=[ctx.id, i]) + logging.info('Added job %d to context.', i) + + # When the Context is exited, the tasks are inserted (if there are no + # errors). + + logging.info('Async jobs for context batch inserted.') + + self.response.out.write('Successfully inserted a group of Async jobs.') + + +def async_worker(*args, **kwargs): + """This function is called by furious tasks to demonstrate usage.""" + logging.info('Context %s, function %s', *args) + + return args + + +def context_complete(context_id): + """Log out that the context is complete.""" + logging.info('Context %s is.......... DONE.', context_id) + + from furious.context import get_current_async_with_context + + _, context = get_current_async_with_context() + + if not context: + logging.error("Could not load context") + return + + for task_id, result in context.result.items(): + logging.info("#########################") + logging.info("Task Id: %s and Result: %s", task_id, result) + + return context_id diff --git a/example/context_events.py b/example/context_events.py new file mode 100644 index 0000000..e05a4c2 --- /dev/null +++ b/example/context_events.py @@ -0,0 +1,68 @@ +# +# Copyright 2014 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Example of using Context events for work flow. + +This example creates a context with events, then adds a few Async jobs to the +Context. +""" + + +import logging + +import webapp2 + + +class ContextEventsHandler(webapp2.RequestHandler): + """Demonstrate using Context Events to make work flows.""" + def get(self): + from furious.async import Async + from furious import context + + count = int(self.request.get('tasks', 5)) + + # Create a new furious Context. + with context.new() as ctx: + # Set a completion event handler. + ctx.set_event_handler('complete', + Async(context_complete, args=[ctx.id])) + + # Insert some Asyncs. + for i in xrange(count): + ctx.add(target=async_worker, args=[ctx.id, i]) + logging.info('Added job %d to context.', i) + + # When the Context is exited, the tasks are inserted (if there are no + # errors). + + logging.info('Async jobs for context batch inserted.') + + self.response.out.write('Successfully inserted a group of Async jobs.') + + +def async_worker(*args, **kwargs): + """This function is called by furious tasks to demonstrate usage.""" + logging.info('Context %s, function %s', *args) + + return args + + +def context_complete(context_id): + """Log out that the context is complete.""" + logging.info('Context %s is.......... DONE.', context_id) + + return context_id + diff --git a/example/context_inherit.py b/example/context_inherit.py new file mode 100644 index 0000000..6c45b79 --- /dev/null +++ b/example/context_inherit.py @@ -0,0 +1,80 @@ +# +# Copyright 2014 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Example of using Context events for work flow. + +This example creates a context and illustrates how the completion checks will +inherit the task queue that each async runs in by having tasks alternate which +queue they will run in. If you observer the queues you will see that tasks +that run in one queue will have their completion checks for the context run +in that queue as well""" + + +import logging + +import webapp2 + + +class ContextInheritHandler(webapp2.RequestHandler): + """Demonstrate using Context Events with queue specific parameters and how + that affects completion checks""" + def get(self): + from furious.async import Async + + count = int(self.request.get('tasks', 5)) + + Async(insert_tasks, queue='example', args=[count]).start() + self.response.out.write('Successfully inserted a group of Async jobs.') + + +def insert_tasks(count): + from furious.async import Async + from furious import context + # Create a new furious Context. + with context.new() as ctx: + # Set a completion event handler. + ctx.set_event_handler('complete', + Async(context_complete, queue='example', + args=[ctx.id])) + + # Insert some Asyncs. The completion check will use each async's queue + for i in xrange(count): + + if i % 2 == 0: + queue = 'example' + else: + queue = 'default' + ctx.add(target=async_worker, queue=queue, args=[ctx.id, i]) + logging.info('Added job %d to context.', i) + + # When the Context is exited, the tasks are inserted (if there are no + # errors). + + logging.info('Async jobs for context batch inserted.') + + +def async_worker(*args, **kwargs): + """This function is called by furious tasks to demonstrate usage.""" + logging.info('Context %s, function %s', *args) + + return args + + +def context_complete(context_id): + """Log out that the context is complete.""" + logging.info('Context %s is.......... DONE.', context_id) + + return context_id diff --git a/example/context_intro.py b/example/context_intro.py new file mode 100644 index 0000000..3ab0d08 --- /dev/null +++ b/example/context_intro.py @@ -0,0 +1,61 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A very basic async context example. + +This example will create a context that adds an Async manually to the context. +It then adds 5 Async jobs through the shorthand style. +""" + + +import logging + +import webapp2 + + +class ContextIntroHandler(webapp2.RequestHandler): + """Demonstrate using a Context to batch insert a group of furious tasks.""" + def get(self): + from furious.async import Async + from furious import context + + # Create a new furious Context. + with context.new() as ctx: + # "Manually" instantiate and add an Async object to the Context. + async_task = Async( + target=example_function, kwargs={'first': 'async'}) + ctx.add(async_task) + logging.info('Added manual job to context.') + + # Use the shorthand style, note that add returns the Async object. + for i in xrange(5): + ctx.add(target=example_function, args=[i]) + logging.info('Added job %d to context.', i) + + # When the Context is exited, the tasks are inserted (if there are no + # errors). + + logging.info('Async jobs for context batch inserted.') + + self.response.out.write('Successfully inserted a group of Async jobs.') + + +def example_function(*args, **kwargs): + """This function is called by furious tasks to demonstrate usage.""" + logging.info('example_function executed with args: %r, kwargs: %r', + args, kwargs) + + return args diff --git a/example/grep.py b/example/grep.py new file mode 100644 index 0000000..2a77d7c --- /dev/null +++ b/example/grep.py @@ -0,0 +1,90 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""This is a Furious example that greps the source code of the application and +logs a message with each line that satisifes the input regular expression.""" + +import logging +import os +import re + +import webapp2 + +from furious import context +from furious.async import Async +from furious.async import defaults + + +class GrepHandler(webapp2.RequestHandler): + """This is the handler that starts off a grep run. It takes the query out + of the query string and starts a new Furious Context to run all of the + Asyncs in.""" + + def get(self): + query = self.request.get('query') + curdir = os.getcwd() + + # create the context and start the first Async + with context.new(): + build_and_start(query, curdir) + + self.response.out.write('starting grep for query: %s' % query) + + +def log_results(): + """This is the callback that is run once the Async task is finished. It + takes the output from grep and logs it.""" + from furious.context import get_current_async + + # Get the recently finished Async object. + async = get_current_async() + + # Pull out the result data and log it. + for result in async.result: + logging.info(result) + + +def build_and_start(query, directory): + """This function will create and then start a new Async task with the + default callbacks argument defined in the decorator.""" + + Async(target=grep, args=[query, directory]).start() + + +def grep_file(query, item): + """This function performs the actual grep on a given file.""" + return ['%s: %s' % (item, line) for line in open(item) + if re.search(query, line)] + + +@defaults(callbacks={'success': log_results}) +def grep(query, directory): + """This function will search through the directory structure of the + application and for each directory it finds it launches an Async task to + run itself. For each .py file it finds, it actually greps the file and then + returns the found output.""" + + dir_contents = os.listdir(directory) + results = [] + for item in dir_contents: + path = os.path.join(directory, item) + if os.path.isdir(path): + build_and_start(query, path) + else: + if item.endswith('.py'): + results.extend(grep_file(query, path)) + return results + diff --git a/example/limits.py b/example/limits.py new file mode 100644 index 0000000..e395e49 --- /dev/null +++ b/example/limits.py @@ -0,0 +1,48 @@ +# +# Copyright 2014 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import time + +import webapp2 + +from furious import context + + +class LimitHandler(webapp2.RequestHandler): + + def get(self): + sleep = self.request.get('sleep', 1) + num = int(self.request.get('num', 1)) + queue = self.request.get('queue', 'default') + + logging.info('sleep: %s', sleep) + logging.info('num: %s', num) + logging.info('queue: %s', queue) + + with context.new() as ctx: + for i in xrange(int(num)): + ctx.add(sleeper, (sleep,), queue=queue) + + self.response.out.write('Successfully inserted a group of Async jobs.') + + +def sleeper(sleep): + logging.info('sleeper before') + logging.info('sleep: %s', sleep) + time.sleep(float(sleep)) + logging.info('sleeper after') + diff --git a/example/runner.py b/example/runner.py new file mode 100644 index 0000000..85304ad --- /dev/null +++ b/example/runner.py @@ -0,0 +1,97 @@ +#!/usr/bin/python +""" +This script will aid in running the examples and "integration" tests included +with furious. + +To run: + + python example/runner.py workflow + +This will hit the /workflow url, causing the "workflow" example to run. +""" + +import argparse +import sys + + +def args(): + """Add and parse the arguments for the script. + + url: the url of the example to run + gae-sdk-path: this allows a user to point the script to their GAE SDK + if it's not in /usr/local/google_appengine. + """ + parser = argparse.ArgumentParser(description='Run the Furious Examples.') + + parser.add_argument('--gae-sdk-path', metavar='S', dest="gae_lib_path", + default="/usr/local/google_appengine", + help='path to the GAE SDK') + + parser.add_argument('url', metavar='U', default="", nargs=1, + help="the endpoint to run") + + return parser.parse_args() + + +def setup(options): + """Grabs the gae_lib_path from the options and inserts it into the first + index of the sys.path. Then calls GAE's fix_sys_path to get all the proper + GAE paths included. + + :param options: + """ + sys.path.insert(0, options.gae_lib_path) + + from dev_appserver import fix_sys_path + fix_sys_path() + + +def run(options): + """Run the passed in url of the example using GAE's rpc runner. + + Uses appengine_rpc.HttpRpcServer to send a request to the url passed in + via the options. + + :param options: + """ + from google.appengine.tools import appengine_rpc + from google.appengine.tools import appcfg + + source = 'furious' + + # use the same user agent that GAE uses in appcfg + user_agent = appcfg.GetUserAgent() + + # Since we're only using the dev server for now we can hard code these + # values. This will need to change and accept these values as variables + # when this is wired up to hit appspots. + server = appengine_rpc.HttpRpcServer( + 'localhost:8080', lambda: ('test@example.com', 'password'), user_agent, + source, secure=False) + + # if no url is passed in just use the top level. + url = "/" + if options.url: + url += options.url[0] + + # use the dev server authentication for now. + server._DevAppServerAuthenticate() + + # send a simple GET request to the url + server.Send(url, content_type="text/html; charset=utf-8", + payload=None) + + +def main(): + """Send a request to the url passed in via the options using GAE's rpc + server. + """ + options = args() + + setup(options) + + run(options) + + +if __name__ == "__main__": + main() diff --git a/example/simple_workflow.py b/example/simple_workflow.py new file mode 100644 index 0000000..d6f92ac --- /dev/null +++ b/example/simple_workflow.py @@ -0,0 +1,57 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A Simple Workflow Example. + +Inserts a Async targetting the simple_state_machine function. This function +does a random number generation. If that number falls above a certain range +it will return another Aysnc pointed at the same simple_state_machine function. +If the simple_state_machine random falls below the range it will return and +complete the simple workflow. +""" + +import logging + +import webapp2 + + +class SimpleWorkflowHandler(webapp2.RequestHandler): + """Demonstrate constructing a simple state machine.""" + def get(self): + from furious.async import Async + + # Instantiate an Async object to start the chain. + Async(target=simple_state_machine).start() + + logging.info('Async chain kicked off.') + + self.response.out.write('Successfully inserted Async chain starter.') + + +def simple_state_machine(): + """Pick a number, if it is more than some cuttoff continue the chain.""" + from random import random + + from furious.async import Async + + number = random() + logging.info('Generating a number... %s', number) + + if number > 0.25: + logging.info('Continuing to do stuff.') + return Async(target=simple_state_machine) + + return number diff --git a/example/templates/batcher.html b/example/templates/batcher.html new file mode 100644 index 0000000..66535f9 --- /dev/null +++ b/example/templates/batcher.html @@ -0,0 +1,191 @@ + + + + + + Async Batcher Example + + +

Async Batcher Example

+ +
+ + +
+
+ + +
+
+ + +
+ + +

Color Stats

+ +
+ + +
+
+ + + diff --git a/furious/__init__.py b/furious/__init__.py index f258ce3..bfbfc62 100644 --- a/furious/__init__.py +++ b/furious/__init__.py @@ -13,3 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # + + +def csrf_check(request): + """ + Throws an HTTP 403 error if a CSRF attack is detected, same logic as the deferred module. + + https://cloud.google.com/appengine/docs/standard/python/refdocs/modules/google/appengine/ext/deferred/deferred + """ + import logging + import webapp2 + + in_prod = ( + not request.environ.get("SERVER_SOFTWARE").startswith("Devel")) + if in_prod and request.environ.get("REMOTE_ADDR") != "0.1.0.2": + logging.error("Detected an attempted CSRF attack from {}. This request did " + "not originate from Task Queue.".format(request.environ.get("REMOTE_ADDR"))) + webapp2.abort(403) diff --git a/furious/_furious.yaml b/furious/_furious.yaml new file mode 100644 index 0000000..2bb5d57 --- /dev/null +++ b/furious/_furious.yaml @@ -0,0 +1,4 @@ +persistence: ndb +cleanupqueue: low-priority +cleanupdelay: 7600 +defaultqueue: default diff --git a/furious/_pkg_meta.py b/furious/_pkg_meta.py new file mode 100644 index 0000000..1499204 --- /dev/null +++ b/furious/_pkg_meta.py @@ -0,0 +1,2 @@ +version_info = (1, 6, 5) +version = '.'.join(map(str, version_info)) diff --git a/furious/async.py b/furious/async.py index 48931c5..a279f2a 100644 --- a/furious/async.py +++ b/furious/async.py @@ -67,35 +67,36 @@ def run_me(*args, **kwargs): 3) options specified in the constructor. 4) options specified by @defaults decorator. """ - +import copy +from functools import partial from functools import wraps - import json +import os +import time +import uuid + +from furious.job_utils import decode_callbacks +from furious.job_utils import encode_callbacks +from furious.job_utils import get_function_path_and_options +from furious.job_utils import path_to_reference +from furious.job_utils import reference_to_path -from .job_utils import get_function_path_and_options +from furious import errors __all__ = ['ASYNC_DEFAULT_QUEUE', 'ASYNC_ENDPOINT', 'Async', 'defaults'] ASYNC_DEFAULT_QUEUE = 'default' -ASYNC_ENDPOINT = '/_ah/queue/async' - - -class NotExecutedError(Exception): - """This Async has not yet been executed.""" - - -class NotExecutingError(Exception): - """This Async in not currently executing.""" - - -class AlreadyExecutedError(Exception): - """This Async has already been executed.""" +ASYNC_ENDPOINT = '/_queue/async' +MAX_DEPTH = 100 +MAX_RESTARTS = 10 +DISABLE_RECURSION_CHECK = -1 +RETRY_SLEEP_SECS = 4 - -class AlreadyExecutingError(Exception): - """This Async is currently executing.""" +DEFAULT_RETRY_OPTIONS = { + 'task_retry_limit': MAX_RESTARTS +} class Async(object): @@ -109,11 +110,19 @@ def __init__(self, target, args=None, kwargs=None, **options): self.update_options(**options) + self._initialize_recursion_depth() + + self._context_id = self._get_context_id() + self._parent_id = self._get_parent_id() + self._id = self._get_id() + self._execution_context = None self._executing = False self._executed = False + self._persistence_engine = None + self._result = None @property @@ -127,11 +136,11 @@ def executing(self): @executing.setter def executing(self, executing): if self._executed: - raise AlreadyExecutedError( - 'You can not execute and executed job.') + raise errors.AlreadyExecutedError( + 'You can not execute an executed job.') if self._executing: - raise AlreadyExecutingError( + raise errors.AlreadyExecutingError( 'Job is already executing, can not set executing.') self._executing = executing @@ -139,7 +148,7 @@ def executing(self, executing): @property def result(self): if not self.executed: - raise NotExecutedError( + raise errors.NotExecutedError( 'You must execute this Async before getting its result.') return self._result @@ -147,16 +156,97 @@ def result(self): @result.setter def result(self, result): if not self._executing: - raise NotExecutingError( + raise errors.NotExecutingError( 'The Async must be executing to set its result.') self._result = result self._executing = False self._executed = True + if self._options.get('persist_result'): + self._persist_result() + + def _persist_result(self): + """Store this Async's result in persistent storage.""" + self._prepare_persistence_engine() + + return self._persistence_engine.store_async_result( + self.id, self.result) + + def _decorate_job(self): + """Returns the job function. + + A subclass may override `Async._decorate_job` in order to wrap the + original target using a decorator function. + """ + function_path = self.job[0] + func = path_to_reference(function_path) + return func + + @property + def function_path(self): + return self.job[0] + @property def _function_path(self): - return self._options['job'][0] + # DEPRECATED: Hanging around for backwards compatibility. + return self.function_path + + @property + def job(self): + """job is stored as a (function path, args, kwargs) tuple.""" + return self._options['job'] + + @property + def recursion_depth(self): + """Get the current recursion depth. `None` indicates uninitialized + recursion info. + """ + recursion_options = self._options.get('_recursion', {}) + return recursion_options.get('current', None) + + def _initialize_recursion_depth(self): + """Ensure recursion info is initialized, if not, initialize it.""" + from furious.context import get_current_async + + recursion_options = self._options.get('_recursion', {}) + + current_depth = recursion_options.get('current', 0) + max_depth = recursion_options.get('max', MAX_DEPTH) + + try: + executing_async = get_current_async() + + # If this async is within an executing async, use the depth off + # that async. Otherwise use the depth set in the async's options. + current_depth = executing_async.recursion_depth + + # If max_depth does not equal MAX_DEPTH, it is custom. Otherwise + # use the max_depth from the containing async. + if max_depth == MAX_DEPTH: + executing_options = executing_async.get_options().get( + '_recursion', {}) + max_depth = executing_options.get('max', max_depth) + + except errors.NotInContextError: + # This Async is not being constructed inside an executing Async. + pass + + # Store the recursion info. + self.update_options(_recursion={'current': current_depth, + 'max': max_depth}) + + def check_recursion_depth(self): + """Check recursion depth, raise AsyncRecursionError if too deep.""" + from furious.async import MAX_DEPTH + + recursion_options = self._options.get('_recursion', {}) + max_depth = recursion_options.get('max', MAX_DEPTH) + + # Check if recursion check has been disabled, then check depth. + if (max_depth != DISABLE_RECURSION_CHECK and + self.recursion_depth > max_depth): + raise errors.AsyncRecursionError('Max recursion depth reached.') def _update_job(self, target, args, kwargs): """Specify the function this async job is to execute when run.""" @@ -173,8 +263,7 @@ def _update_job(self, target, args, kwargs): def set_execution_context(self, execution_context): """Set the ExecutionContext this async is executing under.""" if self._execution_context: - from .context import AlreadyInContextError - raise AlreadyInContextError + raise errors.AlreadyInContextError self._execution_context = execution_context @@ -187,6 +276,13 @@ def update_options(self, **options): _check_options(options) + if 'persistence_engine' in options: + options['persistence_engine'] = reference_to_path( + options['persistence_engine']) + + if 'id' in options: + self._id = options['id'] + self._options.update(options) def get_callbacks(self): @@ -209,66 +305,347 @@ def get_task_args(self): def to_task(self): """Return a task object representing this async job.""" from google.appengine.api.taskqueue import Task + from google.appengine.api.taskqueue import TaskRetryOptions - url = "%s/%s" % (ASYNC_ENDPOINT, self._function_path) + self._increment_recursion_level() + self.check_recursion_depth() + + url = "%s/%s" % (ASYNC_ENDPOINT, self.function_path) kwargs = { 'url': url, 'headers': self.get_headers().copy(), - 'payload': json.dumps(self.to_dict()), + 'payload': json.dumps(self.to_dict()) } - kwargs.update(self.get_task_args()) + kwargs.update(copy.deepcopy(self.get_task_args())) + + # Set task_retry_limit + retry_options = copy.deepcopy(DEFAULT_RETRY_OPTIONS) + retry_options.update(kwargs.pop('retry_options', {})) + kwargs['retry_options'] = TaskRetryOptions(**retry_options) return Task(**kwargs) - def start(self): - """Insert the task into the requested queue, 'default' if non given.""" - from google.appengine.api.taskqueue import Queue + def start(self, transactional=False, async=False, rpc=None): + """Insert the task into the requested queue, 'default' if non given. + + If a TransientError is hit the task will re-insert the task. If a + TaskAlreadyExistsError or TombstonedTaskError is hit the task will + silently fail. + + If the async flag is set, then the add will be done asynchronously and + the return value will be the rpc object; otherwise the return value is + the task itself. If the rpc kwarg is provided, but we're not in async + mode, then it is ignored. + """ + from google.appengine.api import taskqueue task = self.to_task() - Queue(name=self.get_queue()).add(task) - # TODO: Return a "result" object. + queue = taskqueue.Queue(name=self.get_queue()) + retry_transient = self._options.get('retry_transient_errors', True) + retry_delay = self._options.get('retry_delay', RETRY_SLEEP_SECS) - def to_dict(self): - """Return this async job as a dict suitable for json encoding.""" - import copy + add = queue.add + if async: + add = partial(queue.add_async, rpc=rpc) + + try: + ret = add(task, transactional=transactional) + except taskqueue.TransientError: + # Always re-raise for transactional insert, or if specified by + # options. + if transactional or not retry_transient: + raise - options = copy.deepcopy(self._options) + time.sleep(retry_delay) - # JSON don't like datetimes. - eta = options.get('task_args', {}).get('eta') - if eta: - import time + ret = add(task, transactional=transactional) + except (taskqueue.TaskAlreadyExistsError, + taskqueue.TombstonedTaskError): + return - options['task_args']['eta'] = time.mktime(eta.timetuple()) + # TODO: Return a "result" object. + return ret - callbacks = self._options.get('callbacks') - if callbacks: - options['callbacks'] = _encode_callbacks(callbacks) + def __deepcopy__(self, *args): + """In order to support callbacks being Async objects, we need to + support being deep copied. + """ + return self - return options + def to_dict(self): + """Return this async job as a dict suitable for json encoding.""" + return encode_async_options(self) @classmethod def from_dict(cls, async): """Return an async job from a dict output by Async.to_dict.""" + async_options = decode_async_options(async) + + target, args, kwargs = async_options.pop('job') - async_options = async.copy() + return cls(target, args, kwargs, **async_options) - # JSON don't like datetimes. - eta = async_options.get('task_args', {}).get('eta') - if eta: - from datetime import datetime + def _prepare_persistence_engine(self): + """Load the specified persistence engine, or the default if none is + set. + """ + if self._persistence_engine: + return - async_options['task_args']['eta'] = datetime.fromtimestamp(eta) + persistence_engine = self._options.get('persistence_engine') + if persistence_engine: + self._persistence_engine = path_to_reference(persistence_engine) + return - target, args, kwargs = async_options.pop('job') + from furious.config import get_default_persistence_engine - # If there are callbacks, reconstitute them. - callbacks = async_options.get('callbacks', {}) - if callbacks: - async_options['callbacks'] = _decode_callbacks(callbacks) + self._persistence_engine = get_default_persistence_engine() - return Async(target, args, kwargs, **async_options) + def _get_context_id(self): + """If this async is in a context set the context id.""" + + from furious.context import get_current_context + + context_id = self._options.get('context_id') + + if context_id: + return context_id + + try: + context = get_current_context() + except errors.NotInContextError: + context = None + self.update_options(context_id=None) + + if context: + context_id = context.id + self.update_options(context_id=context_id) + + return context_id + + def _get_parent_id(self): + """If this async is in within another async set that async id as the + parent. + """ + parent_id = self._options.get('parent_id') + if parent_id: + return parent_id + + from furious.context import get_current_async + + try: + async = get_current_async() + except errors.NotInContextError: + async = None + + if async: + parent_id = ":".join([async.parent_id.split(":")[0], async.id]) + else: + parent_id = self.request_id + + self.update_options(parent_id=parent_id) + + return parent_id + + def _get_id(self): + """If this async has no id, generate one.""" + id = self._options.get('id') + if id: + return id + + id = uuid.uuid4().hex + self.update_options(id=id) + return id + + @property + def id(self): + """Return this Async's ID value.""" + return self._id + + @property + def context_id(self): + """Return this Async's Context Id if it exists.""" + return self._context_id + + @property + def parent_id(self): + """Return this Async's Parent Id if it exists.""" + return self._parent_id + + @property + def full_id(self): + """Return the full_id for this Async. Consists of the parent id, id and + context id. + """ + full_id = "" + + if self.parent_id: + full_id = ":".join([self.parent_id, self.id]) + else: + full_id = self.id + + if self.context_id: + full_id = "|".join([full_id, self.context_id]) + + return full_id + + @property + def request_id(self): + return os.environ.get('REQUEST_LOG_ID', uuid.uuid4().hex) + + def _increment_recursion_level(self): + """Increment current_depth based on either defaults or the enclosing + Async. + """ + # Update the recursion info. This is done so that if an async created + # outside an executing context, or one previously created is later + # loaded from storage, that the "current" setting is correctly set. + self._initialize_recursion_depth() + + recursion_options = self._options.get('_recursion', {}) + current_depth = recursion_options.get('current', 0) + 1 + max_depth = recursion_options.get('max', MAX_DEPTH) + + # Increment and store + self.update_options(_recursion={'current': current_depth, + 'max': max_depth}) + + @property + def context_id(self): + """Return this Async's Context Id if it exists.""" + if not self._context_id: + self._context_id = self._get_context_id() + self.update_options(context_id=self._context_id) + + return self._context_id + + def _get_context_id(self): + """If this async is in a context set the context id.""" + + from furious.context import get_current_context + + context_id = self._options.get('context_id') + + if context_id: + return context_id + + try: + context = get_current_context() + except errors.NotInContextError: + context = None + self.update_options(context_id=None) + + if context: + context_id = context.id + self.update_options(context_id=context_id) + + return context_id + + +class AsyncResult(object): + + SUCCESS = 1 + ERROR = 2 + ABORT = 3 + + def __init__(self, payload=None, status=None): + self.payload = payload + self.status = status + + @property + def success(self): + """Return True if the status is a success. This is true if the status + is not an error state. So abort or success. Abort is considered a + success as it's something expected by the developer. Errors would + generally only happen in tasks if something unexpected occurred. + """ + return self.status != self.ERROR + + def to_dict(self): + """Return the AsyncResult converted to a dictionary and also to an + serializable format. + """ + return { + 'status': self.status, + 'payload': self._payload_to_dict() + } + + def _payload_to_dict(self): + """When an error status the payload is holding an AsyncException that + is converted to a serializable dict. + """ + if self.status != self.ERROR or not self.payload: + return self.payload + + import traceback + + return { + "error": self.payload.error, + "args": self.payload.args, + "traceback": traceback.format_exception(*self.payload.traceback) + } + + +def async_from_options(options): + """Deserialize an Async or Async subclass from an options dict.""" + _type = options.pop('_type', 'furious.async.Async') + + _type = path_to_reference(_type) + + return _type.from_dict(options) + + +def encode_async_options(async): + """Encode Async options for JSON encoding.""" + options = copy.deepcopy(async._options) + + options['_type'] = reference_to_path(async.__class__) + + # JSON don't like datetimes. + eta = options.get('task_args', {}).get('eta') + if eta: + options['task_args']['eta'] = time.mktime(eta.timetuple()) + + callbacks = async._options.get('callbacks') + if callbacks: + options['callbacks'] = encode_callbacks(callbacks) + + if '_context_checker' in options: + _checker = options.pop('_context_checker') + options['__context_checker'] = reference_to_path(_checker) + + if '_process_results' in options: + _processor = options.pop('_process_results') + options['__process_results'] = reference_to_path(_processor) + + return options + + +def decode_async_options(options): + """Decode Async options from JSON decoding.""" + async_options = copy.deepcopy(options) + + # JSON don't like datetimes. + eta = async_options.get('task_args', {}).get('eta') + if eta: + from datetime import datetime + + async_options['task_args']['eta'] = datetime.fromtimestamp(eta) + + # If there are callbacks, reconstitute them. + callbacks = async_options.get('callbacks', {}) + if callbacks: + async_options['callbacks'] = decode_callbacks(callbacks) + + if '__context_checker' in options: + _checker = options['__context_checker'] + async_options['_context_checker'] = path_to_reference(_checker) + + if '__process_results' in options: + _processor = options['__process_results'] + async_options['_process_results'] = path_to_reference(_processor) + return async_options def defaults(**options): @@ -296,39 +673,3 @@ def _check_options(options): return assert 'job' not in options - #assert 'callbacks' not in options - - -def _encode_callbacks(callbacks): - """Encode callbacks to as a dict suitable for JSON encoding.""" - if not callbacks: - return - - encoded_callbacks = {} - for event, callback in callbacks.iteritems(): - if callable(callback): - callback, _ = get_function_path_and_options(callback) - - elif isinstance(callback, Async): - callback = callback.to_dict() - - encoded_callbacks[event] = callback - - return encoded_callbacks - - -def _decode_callbacks(encoded_callbacks): - """Decode the callbacks to an executable form.""" - from furious.job_utils import function_path_to_reference - - callbacks = {} - for event, callback in encoded_callbacks.iteritems(): - if isinstance(callback, dict): - callback = Async.from_dict(callback) - else: - callback = function_path_to_reference(callback) - - callbacks[event] = callback - - return callbacks - diff --git a/furious/batcher.py b/furious/batcher.py index 173eb7c..599912c 100644 --- a/furious/batcher.py +++ b/furious/batcher.py @@ -1,12 +1,32 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import json +import logging import time +import uuid from google.appengine.api import memcache +from google.appengine.runtime.apiproxy_errors import DeadlineExceededError -from .async import Async +from furious.async import Async -MESSAGE_DEFAULT_QUEUE = 'default_pull' +MESSAGE_DEFAULT_QUEUE = 'default-pull' MESSAGE_PROCESSOR_NAME = 'processor' +MESSAGE_BATCH_NAME = 'agg-batch' METHOD_TYPE = 'PULL' @@ -17,6 +37,8 @@ def __init__(self, **options): self.update_options(**options) + self._id = self._get_id() + def get_options(self): """Return this message's configuration options.""" return self._options @@ -59,6 +81,7 @@ def insert(self): from google.appengine.api.taskqueue import Queue task = self.to_task() + Queue(name=self.get_queue()).add(task) def to_dict(self): @@ -74,6 +97,21 @@ def to_dict(self): return options + def _get_id(self): + """If this message has no id, generate one.""" + id = self._options.get('id') + if id: + return id + + id = uuid.uuid4().hex + self.update_options(id=id) + return id + + @property + def id(self): + """Return this Message's ID value.""" + return self._id + @classmethod def from_dict(cls, message): """Return an message from a dict output by Async.to_dict.""" @@ -121,7 +159,7 @@ def to_task(self): @property def group_key(self): """Return the :class: `str` group key based off of the tag.""" - return 'agg-batch-%s' % (self.tag) + return "%s-%s" % (MESSAGE_BATCH_NAME, self.tag) @property def current_batch(self): @@ -148,9 +186,135 @@ def time_throttle(self): return int(time.time() / max(1, self.frequency)) -def fetch_messages(): - pass +class MessageIterator(object): + """This iterator will return a batch of messages for a given group. + + This iterator should be directly used when trying to avoid the lease + operation inside a transaction, or when other flows are needed. + """ + + def __init__(self, tag, queue_name, size, duration=60, deadline=10, + auto_delete=True): + """The generator will yield json deserialized payloads from tasks with + the corresponding tag. + :param tag: :class: `str` Pull queue tag to query against + :param queue_name: :class: `str` Name of PULL queue holding tasks to + lease. + :param size: :class: `int` The number of items to pull at once + :param duration: :class: `int` After this time, the tasks may be leased + again. Tracked in seconds + :param deadline: :class: `int` The time in seconds to wait for the rpc. + :param auto_delete: :class: `bool` Delete tasks when iteration is + complete. + + :return: :class: `iterator` of json deserialized payloads + """ + from google.appengine.api.taskqueue import Queue + + self.queue_name = queue_name + self.queue = Queue(name=self.queue_name) + + self.tag = tag + self.size = size + self.duration = duration + self.auto_delete = auto_delete + self.deadline = deadline + + self._messages = [] + self._processed_messages = [] + self._fetched = False + + def fetch_messages(self): + """Fetch messages from the specified pull-queue. + + This should only be called a single time by a given MessageIterator + object. If the MessageIterator is iterated over again, it should + return the originally leased messages. + """ + if self._fetched: + return -def bump_batch(): - pass + start = time.time() + + loaded_messages = self.queue.lease_tasks_by_tag( + self.duration, self.size, tag=self.tag, deadline=self.deadline) + + # If we are within 0.1 sec of our deadline and no messages were + # returned, then we are hitting queue contention issues and this + # should be a DeadlineExceederError. + # TODO: investigate other ways around this, perhaps async leases, etc. + if (not loaded_messages and + round(time.time() - start, 1) >= self.deadline - 0.1): + raise DeadlineExceededError() + + self._messages.extend(loaded_messages) + + self._fetched = True + + logging.debug("Calling fetch messages with %s:%s:%s:%s:%s:%s" % ( + len(self._messages), len(loaded_messages), + len(self._processed_messages), self.duration, self.size, self.tag)) + + def __iter__(self): + """Initialize this MessageIterator for iteration. + + If messages have not been fetched, fetch them. If messages have been + fetched, reset self._messages and self._processed_messages for + re-iteration. The reset is done to prevent deleting messages that were + never applied. + """ + if self._processed_messages: + # If the iterator is used within a transaction, and there is a + # retry we need to re-process the original messages, not new + # messages. + self._messages = list( + set(self._messages) | set(self._processed_messages)) + self._processed_messages = [] + + if not self._messages: + self.fetch_messages() + + return self + + def next(self): + """Get the next batch of messages from the previously fetched messages. + + If there's no more messages, check if we should auto-delete the + messages and raise StopIteration. + """ + if not self._messages: + if self.auto_delete: + self.delete_messages() + raise StopIteration + + message = self._messages.pop(0) + self._processed_messages.append(message) + return json.loads(message.payload) + + def delete_messages(self, only_processed=True): + """Delete the messages previously leased. + + Unless otherwise directed, only the messages iterated over will be + deleted. + """ + messages = self._processed_messages + if not only_processed: + messages += self._messages + + if messages: + try: + self.queue.delete_tasks(messages) + except Exception: + logging.exception("Error deleting messages") + raise + + +def bump_batch(work_group): + """Return the incremented batch id for the work group + :param work_group: :class: `str` + + :return: :class: `int` current batch id. + """ + key = "%s-%s" % (MESSAGE_BATCH_NAME, work_group) + return memcache.incr(key) diff --git a/furious/config.py b/furious/config.py new file mode 100644 index 0000000..b5908d9 --- /dev/null +++ b/furious/config.py @@ -0,0 +1,220 @@ +# +# Copyright 2013 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +import os + +import yaml + +FURIOUS_YAML_NAMES = ['furious.yaml', 'furious.yml'] + +PERSISTENCE_MODULES = { + 'ndb': 'furious.extras.appengine.ndb_persistence' +} + + +class BadModulePathError(Exception): + """Invalid module path.""" + + +class InvalidPersistenceModuleName(Exception): + """There is no persistence strategy by that name.""" + + +class InvalidYamlFile(Exception): + """The furious.yaml file is invalid yaml.""" + + +class EmptyYamlFile(Exception): + """The furious.yaml file is empty.""" + + +class MissingYamlFile(Exception): + """furious.yaml cannot be found.""" + + +def get_default_persistence_engine(known_modules=PERSISTENCE_MODULES): + """Return the default persistence engine set in furious.yaml.""" + return _get_configured_module('persistence', known_modules=known_modules) + + +def get_completion_cleanup_queue(): + """Get the default queue that completion should use to cleanup markers on. + """ + config = get_config() + return config.get('cleanupqueue') + + +def get_completion_default_queue(): + """Get the default queue that completion should use to cleanup markers on. + """ + config = get_config() + return config.get('defaultqueue') + + +def get_completion_cleanup_delay(): + """Get the default queue that completion should use to cleanup markers on. + """ + config = get_config() + return config.get('cleanupdelay') + + +def get_csrf_check(): + """Get the CSRF check function, which takes one arg (webapp2.Reqeust) + and returns None, throwing an exception if a CSRF attack is detected. + """ + from furious.job_utils import path_to_reference + + return path_to_reference(get_config().get('csrf_check')) + + +def _get_configured_module(option_name, known_modules=None): + """Get the module specified by the value of option_name. The value of the + configuration option will be used to load the module by name from the known + module list or treated as a path if not found in known_modules. + Args: + option_name: name of persistence module + known_modules: dictionary of module names and module paths, + ie: {'ndb':'furious.extras.appengine.ndb_persistence'} + Returns: + module of the module path matching the name in known_modules + """ + from furious.job_utils import path_to_reference + + config = get_config() + option_value = config[option_name] + + # If no known_modules were give, make it an empty dict. + if not known_modules: + known_modules = {} + + module_path = known_modules.get(option_value) or option_value + return path_to_reference(module_path) + + +def find_furious_yaml(config_file=__file__): + """ + Traverse directory trees to find a furious.yaml file + + Begins with the location of this file then checks the + working directory if not found + + Args: + config_file: location of this file, override for + testing + Returns: + the path of furious.yaml or None if not found + """ + checked = set() + result = _find_furious_yaml(os.path.dirname(config_file), checked) + if not result: + result = _find_furious_yaml(os.getcwd(), checked) + return result + + +def _find_furious_yaml(start, checked): + """Traverse the directory tree identified by start + until a directory already in checked is encountered or the path + of furious.yaml is found. + + Checked is present both to make the loop termination easy + to reason about and so the same directories do not get + rechecked + + Args: + start: the path to start looking in and work upward from + checked: the set of already checked directories + + Returns: + the path of the furious.yaml file or None if it is not found + """ + directory = start + while directory not in checked: + checked.add(directory) + for fs_yaml_name in FURIOUS_YAML_NAMES: + yaml_path = os.path.join(directory, fs_yaml_name) + if os.path.exists(yaml_path): + return yaml_path + directory = os.path.dirname(directory) + return None + + +def default_config(): + """The default configuration allows furious to work + even without a user furious.yaml + + Returns: + dictionary of defaults used by various parts of furious + """ + return {'secret_key': + '931b8-i-f44330b4a5-am-3b9b733f-not-secure-043e96882', + 'persistence': 'ndb', + 'cleanupqueue': 'default', + 'cleanupdelay': 7600, + 'defaultqueue': 'default', + 'task_system': 'appengine_taskqueue', + 'csrf_check': 'furious.csrf_check'} + + +def _load_yaml_config(path=None): + """Open and return the yaml contents.""" + furious_yaml_path = path or find_furious_yaml() + if furious_yaml_path is None: + logging.debug("furious.yaml not found.") + return None + + with open(furious_yaml_path) as yaml_file: + return yaml_file.read() + + +def _parse_yaml_config(config_data=None): + """ + Gets the configuration from the found furious.yaml + file and parses the data. + Returns: + a dictionary parsed from the yaml file + """ + data_map = default_config() + + # If we were given config data to use, use it. Otherwise, see if there is + # a furious.yaml to read the config from. Note that the empty string will + # result in the default config being used. + if config_data is None: + config_data = _load_yaml_config() + + if not config_data: + logging.debug("No custom furious config, using default config.") + return data_map + + # TODO: validate the yaml contents + config = yaml.safe_load(config_data) + + # If there was a valid custom config, it will be a dict. Otherwise, + # ignore it. + if isinstance(config, dict): + # Apply the custom config over the default config. This allows us to + # extend functionality without breaking old stuff. + data_map.update(config) + elif not None: + raise InvalidYamlFile("The furious.yaml file " + "is invalid yaml") + + return data_map + + +def get_config(): + return _config + +_config = _parse_yaml_config() diff --git a/furious/context/__init__.py b/furious/context/__init__.py index e8a1e47..9313a05 100644 --- a/furious/context/__init__.py +++ b/furious/context/__init__.py @@ -38,31 +38,28 @@ """ -from . import _local -from .context import Context +from furious.context import _local +from furious.context.auto_context import AutoContext +from furious.context.context import Context -from . import _execution +from furious.context import _execution + +from furious import errors -ContextExistsError = _execution.ContextExistsError -CorruptContextError = _execution.CorruptContextError execution_context_from_async = _execution.execution_context_from_async -class AlreadyInContextError(Exception): - """Attempt to set context on an Async that is already executing in a - context. +def new(batch_size=None, **options): + """Get a new furious context and add it to the registry. If a batch size is + specified, use an AutoContext which inserts tasks in batches as they are + added to the context. """ - -class NotInContextError(Exception): - """Call that requires context made outside context.""" - - -def new(): - """Get a new furious context and add it to the registry.""" - - new_context = Context() + if batch_size: + new_context = AutoContext(batch_size=batch_size, **options) + else: + new_context = Context(**options) _local.get_local_context().registry.append(new_context) @@ -78,5 +75,31 @@ def get_current_async(): if local_context._executing_async: return local_context._executing_async[-1] - raise NotInContextError('Not in an _ExecutionContext.') + raise errors.NotInContextError('Not in an _ExecutionContext.') + + +def get_current_context(): + """Return a reference to the current Context object. + """ + local_context = _local.get_local_context() + + if local_context.registry: + return local_context.registry[-1] + + raise errors.NotInContextError('Not in a Context.') + + +def get_current_async_with_context(): + """Return a reference to the currently executing Async job object and it's + triggering context. Return None for the async if not in an Async job and + None for the context if the Async was not triggered within a context. + """ + async = get_current_async() + + if not async: + return None, None + + if not async.context_id: + return async, None + return async, Context.load(async.context_id) diff --git a/furious/context/_execution.py b/furious/context/_execution.py index aed544d..09b0567 100644 --- a/furious/context/_execution.py +++ b/furious/context/_execution.py @@ -29,23 +29,12 @@ """ -from . import _local +from furious.context import _local +from furious import errors -__all__ = ["ContextExistsError", - "CorruptContextError", - "execution_context_from_async"] - -class ContextExistsError(Exception): - """Call made within context that should not be.""" - - -class CorruptContextError(Exception): - """ExecutionContext raised when the execution context stack is corrupted. - """ - def __init__(self, *exc_info): - self.exc_info = exc_info +__all__ = ["execution_context_from_async"] def execution_context_from_async(async): @@ -55,7 +44,7 @@ def execution_context_from_async(async): local_context = _local.get_local_context() if local_context._executing_async_context: - raise ContextExistsError + raise errors.ContextExistsError execution_context = _ExecutionContext(async) local_context._executing_async_context = execution_context @@ -68,7 +57,7 @@ class _ExecutionContext(object): """ def __init__(self, async): """Initialize a context with an async task.""" - from ..async import Async + from furious.async import Async if not isinstance(async, Async): raise TypeError("async must be an Async instance.") @@ -93,7 +82,7 @@ def __exit__(self, *exc_info): last = local_context._executing_async.pop() if last is not self._async: local_context._executing_async.append(last) - raise CorruptContextError(*exc_info) + raise errors.CorruptContextError(*exc_info) return False diff --git a/furious/context/_local.py b/furious/context/_local.py index 7be1058..7a3253c 100644 --- a/furious/context/_local.py +++ b/furious/context/_local.py @@ -52,16 +52,12 @@ def _init(): NOTE: Do not directly run this method. """ - global _local_context - # If there is a context and it is initialized to this request, # return, otherwise reinitialize the _local_context. if (hasattr(_local_context, '_initialized') and - _local_context._initialized == os.environ['REQUEST_ID_HASH']): + _local_context._initialized == os.environ.get('REQUEST_ID_HASH')): return - _local_context = threading.local() - # Used to track the context object stack. _local_context.registry = [] @@ -70,12 +66,24 @@ def _init(): _local_context._executing_async = [] # So that we do not inadvertently reinitialize the local context. - _local_context._initialized = os.environ['REQUEST_ID_HASH'] + _local_context._initialized = os.environ.get('REQUEST_ID_HASH') return _local_context +def _clear_context(): + """Clear the context. + + Create a clean uninitialized context. + This is not typically used. It is mainly for use in local tests when + running asyncs within a single process. + """ + + global _local_context + _local_context = threading.local() + + # NOTE: Do not import this directly. If you MUST use this, access it # through get_local_context. -_local_context = None +_local_context = threading.local() diff --git a/furious/context/auto_context.py b/furious/context/auto_context.py new file mode 100644 index 0000000..12dbd25 --- /dev/null +++ b/furious/context/auto_context.py @@ -0,0 +1,87 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Furious AutoContext is used batch inserts of tasks into groups. + +It is similar to Context, but inserts automatically before the context +is exited. +""" + + +from furious.context.context import Context + + +class AutoContext(Context): + """Similar to context, but automatically inserts tasks asynchronously as + they are added to the context. Inserted in batches if specified. + """ + + def __init__(self, batch_size=None, **options): + """Setup this context in addition to accepting a batch_size.""" + + Context.__init__(self, **options) + + self.batch_size = batch_size + + def add(self, target, args=None, kwargs=None, **options): + """Add an Async job to this context. + + Like Context.add(): creates an Async and adds it to our list of tasks. + but also calls _auto_insert_check() to add tasks to queues + automatically. + """ + + # In superclass, add new task to our list of tasks + target = super( + AutoContext, self).add(target, args, kwargs, **options) + + self._auto_insert_check() + + return target + + def _auto_insert_check(self): + """Automatically insert tasks asynchronously. + Depending on batch_size, insert or wait until next call. + """ + + if not self.batch_size: + return + + if len(self._tasks) >= self.batch_size: + self._handle_tasks() + + def _handle_tasks(self): + """Convert Async's into tasks, then insert them into queues. + Similar to the default _handle_tasks, but don't mark all + tasks inserted. + """ + + self._handle_tasks_insert(batch_size=self.batch_size) + self._tasks = [] + + def __exit__(self, exc_type, exc_val, exc_tb): + """In addition to the default __exit__(), also mark all tasks + inserted. + """ + + super(AutoContext, self).__exit__(exc_type, exc_val, exc_tb) + + # Mark all tasks inserted. + self._tasks_inserted = True + + return False + diff --git a/furious/context/context.py b/furious/context/context.py index d5de125..236cfeb 100644 --- a/furious/context/context.py +++ b/furious/context/context.py @@ -41,12 +41,19 @@ queue='workgroup') """ +import abc +import time +import uuid +from furious.job_utils import decode_callbacks +from furious.job_utils import encode_callbacks +from furious.job_utils import path_to_reference +from furious.job_utils import reference_to_path -class ContextAlreadyStartedError(Exception): - """Attempt to set context on an Async that is already executing in a - context. - """ +from furious import errors + +DEFAULT_TASK_BATCH_SIZE = 100 +RETRY_SLEEP_SECS = 4 class Context(object): @@ -55,12 +62,58 @@ class Context(object): NOTE: Use the module's new function to get a context, do not manually instantiate. """ - def __init__(self, insert_tasks=None): + def __init__(self, **options): self._tasks = [] + self._tasks_inserted = False + self._insert_success_count = 0 + self._insert_failed_count = 0 - self.insert_tasks = insert_tasks or _insert_tasks + self._persistence_engine = options.get('persistence_engine', None) + if self._persistence_engine: + options['persistence_engine'] = reference_to_path( + self._persistence_engine) - self._tasks_inserted = False + self._options = options + + if '_task_ids' not in self._options: + self._options['_task_ids'] = [] + + self._id = self._get_id() + self._result = None + + self._insert_tasks = options.pop('insert_tasks', _insert_tasks) + if not callable(self._insert_tasks): + raise TypeError('You must provide a valid insert_tasks function.') + + def _get_id(self): + """If this async has no id, generate one.""" + id = self._options.get('id') + if id: + return id + + id = uuid.uuid4().hex + self._options['id'] = id + return id + + @property + def id(self): + return self._id + + @property + def task_ids(self): + return self._options['_task_ids'] + + @property + def insert_success(self): + return self._insert_success_count + + @property + def insert_failed(self): + return self._insert_failed_count + + @property + def persist_async_results(self): + return self._options.get('persist_async_results', False) def __enter__(self): return self @@ -71,73 +124,320 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False - def _handle_tasks(self): + def _handle_tasks_insert(self, batch_size=None): """Convert all Async's into tasks, then insert them into queues.""" if self._tasks_inserted: - raise ContextAlreadyStartedError( + raise errors.ContextAlreadyStartedError( "This Context has already had its tasks inserted.") task_map = self._get_tasks_by_queue() + + # QUESTION: Should the persist happen before or after the task + # insertion? I feel like this is something that will alter the + # behavior of the tasks themselves by adding a callback (check context + # complete) to each Async's callback stack. + + # If we are able to and there is a reason to persist... persist. + callbacks = self._options.get('callbacks') + if self._persistence_engine and callbacks: + self.persist() + + retry_transient = self._options.get('retry_transient_errors', True) + retry_delay = self._options.get('retry_delay', RETRY_SLEEP_SECS) + for queue, tasks in task_map.iteritems(): - self.insert_tasks(tasks, queue=queue) + for batch in _task_batcher(tasks, batch_size=batch_size): + inserted = self._insert_tasks( + batch, queue=queue, retry_transient_errors=retry_transient, + retry_delay=retry_delay + ) + if isinstance(inserted, (int, long)): + # Don't blow up on insert_tasks that don't return counts. + self._insert_success_count += inserted + self._insert_failed_count += len(batch) - inserted + + def _handle_tasks(self): + """Convert all Async's into tasks, then insert them into queues. + Also mark all tasks inserted to ensure they are not reinserted later. + """ + self._handle_tasks_insert() self._tasks_inserted = True def _get_tasks_by_queue(self): """Return the tasks for this Context, grouped by queue.""" task_map = {} + _checker = None + + # Ask the persistence engine for an Async to use for checking if the + # context is complete. + if self._persistence_engine: + _checker = self._persistence_engine.context_completion_checker for async in self._tasks: queue = async.get_queue() + if _checker: + async.update_options(_context_checker=_checker) + task = async.to_task() task_map.setdefault(queue, []).append(task) return task_map + def _prepare_persistence_engine(self): + """Load the specified persistence engine, or the default if none is + set. + """ + + from furious.config import get_default_persistence_engine + + if self._persistence_engine: + return + + persistence_engine = self._options.get('persistence_engine') + if persistence_engine: + self._persistence_engine = path_to_reference(persistence_engine) + return + + self._persistence_engine = get_default_persistence_engine() + + def set_event_handler(self, event, handler): + """Add an Async to be run on event.""" + # QUESTION: Should we raise an exception if `event` is not in some + # known event-type list? + + self._prepare_persistence_engine() + + callbacks = self._options.get('callbacks', {}) + callbacks[event] = handler + self._options['callbacks'] = callbacks + + def exec_event_handler(self, event, transactional=False): + """Execute the Async set to be run on event.""" + # QUESTION: Should we raise an exception if `event` is not in some + # known event-type list? + + callbacks = self._options.get('callbacks', {}) + + handler = callbacks.get(event) + + if not handler: + raise Exception('Handler not defined!!!') + + handler.start(transactional=transactional) + def add(self, target, args=None, kwargs=None, **options): """Add an Async job to this context. - Takes an Async object or the argumnets to construct an Async - object as argumnets. Returns the newly added Async object. + Takes an Async object or the arguments to construct an Async + object as arguments. Returns the newly added Async object. """ - from ..async import Async + from furious.async import Async + from furious.batcher import Message if self._tasks_inserted: - raise ContextAlreadyStartedError( + raise errors.ContextAlreadyStartedError( "This Context has already had its tasks inserted.") - if not isinstance(target, Async): + if not isinstance(target, (Async, Message)): target = Async(target, args, kwargs, **options) + target.update_options(_context_id=self.id) + + if self.persist_async_results: + target.update_options(persist_result=True) + self._tasks.append(target) + self._options['_task_ids'].append(target.id) return target def start(self): - """Insert this Context's tasks executing.""" + """Insert this Context's tasks so they start executing.""" if self._tasks: self._handle_tasks() + def persist(self): + """Store the context.""" + if not self._persistence_engine: + raise RuntimeError( + 'Specify a valid persistence_engine to persist this context.') + + return self._persistence_engine.store_context(self) + + @classmethod + def load(cls, context_id, persistence_engine=None): + """Load and instantiate a Context from the persistence_engine.""" + if not persistence_engine: + from furious.config import get_default_persistence_engine + persistence_engine = get_default_persistence_engine() -def _insert_tasks(tasks, queue, transactional=False): + if not persistence_engine: + raise RuntimeError( + 'Specify a valid persistence_engine to load the context.') + + return persistence_engine.load_context(context_id) + + def to_dict(self): + """Return this Context as a dict suitable for json encoding.""" + import copy + + options = copy.deepcopy(self._options) + + if self._insert_tasks: + options['insert_tasks'] = reference_to_path(self._insert_tasks) + + if self._persistence_engine: + options['persistence_engine'] = reference_to_path( + self._persistence_engine) + + options.update({ + '_tasks_inserted': self._tasks_inserted, + }) + + callbacks = self._options.get('callbacks') + if callbacks: + options['callbacks'] = encode_callbacks(callbacks) + + return options + + @classmethod + def from_dict(cls, context_options_dict): + """Return a context job from a dict output by Context.to_dict.""" + import copy + + context_options = copy.deepcopy(context_options_dict) + + tasks_inserted = context_options.pop('_tasks_inserted', False) + + insert_tasks = context_options.pop('insert_tasks', None) + if insert_tasks: + context_options['insert_tasks'] = path_to_reference(insert_tasks) + + # The constructor expects a reference to the persistence engine. + persistence_engine = context_options.pop('persistence_engine', None) + if persistence_engine: + context_options['persistence_engine'] = path_to_reference( + persistence_engine) + + # If there are callbacks, reconstitute them. + callbacks = context_options.pop('callbacks', None) + if callbacks: + context_options['callbacks'] = decode_callbacks(callbacks) + + context = cls(**context_options) + + context._tasks_inserted = tasks_inserted + + return context + + @property + def result(self): + """Return the context result object pulled from the persistence_engine + if it has been set. + """ + if not self._result: + if not self._persistence_engine: + return None + + self._result = self._persistence_engine.get_context_result(self) + + return self._result + + +class ContextResultBase(object): + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def items(self): + """Yield the async reuslts for the context.""" + return + + @abc.abstractmethod + def has_errors(self): + """Return the error flag from the completion engine.""" + return + + @abc.abstractmethod + def values(self): + """Return the async reuslt values for the context.""" + return + + +def _insert_tasks(tasks, queue, transactional=False, + retry_transient_errors=True, + retry_delay=RETRY_SLEEP_SECS): """Insert a batch of tasks into the specified queue. If an error occurs during insertion, split the batch and retry until they are successfully - inserted. + inserted. Return the number of successfully inserted tasks. """ from google.appengine.api import taskqueue if not tasks: - return + return 0 try: taskqueue.Queue(name=queue).add(tasks, transactional=transactional) - except (taskqueue.TransientError, + return len(tasks) + except (taskqueue.BadTaskStateError, taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError): - count = len(tasks) - if count <= 1: - return + if len(tasks) <= 1: + # Task has already been inserted, no reason to report an error here. + return 0 + + # If a list of more than one Tasks is given, a raised exception does + # not guarantee that no tasks were added to the queue (unless + # transactional is set to True). To determine which tasks were + # successfully added when an exception is raised, check the + # Task.was_enqueued property. + reinsert = _tasks_to_reinsert(tasks, transactional) + count = len(reinsert) + inserted = len(tasks) - count + inserted += _insert_tasks(reinsert[:count / 2], queue, transactional, + retry_transient_errors, retry_delay) + inserted += _insert_tasks(reinsert[count / 2:], queue, transactional, + retry_transient_errors, retry_delay) + + return inserted + except taskqueue.TransientError: + # Always re-raise for transactional insert, or if specified by + # options. + if transactional or not retry_transient_errors: + raise + + reinsert = _tasks_to_reinsert(tasks, transactional) + + # Retry with a delay, and then let any errors re-raise. + time.sleep(retry_delay) + + taskqueue.Queue(name=queue).add(reinsert, transactional=transactional) + return len(tasks) + + +def _tasks_to_reinsert(tasks, transactional): + """Return a list containing the tasks that should be reinserted based on the + was_enqueued property and whether the insert is transactional or not. + """ + if transactional: + return tasks + + return [task for task in tasks if not task.was_enqueued] + + +def _task_batcher(tasks, batch_size=None): + """Batches large task lists into groups of 100 so that they can all be + inserted. + """ + from itertools import izip_longest + + if not batch_size: + batch_size = DEFAULT_TASK_BATCH_SIZE + + # Ensure the batch size is under the task api limit. + batch_size = min(batch_size, 100) - _insert_tasks(tasks[:count / 2], queue, transactional) - _insert_tasks(tasks[count / 2:], queue, transactional) + args = [iter(tasks)] * batch_size + return ([task for task in group if task] for group in izip_longest(*args)) diff --git a/furious/errors.py b/furious/errors.py new file mode 100644 index 0000000..22d0f58 --- /dev/null +++ b/furious/errors.py @@ -0,0 +1,83 @@ +# +# Copyright 2013 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class BadObjectPathError(Exception): + """Invalid object path.""" + + +class AsyncError(Exception): + """The base class other Async errors can subclass.""" + + +class NotExecutedError(Exception): + """This Async has not yet been executed.""" + + +class NotExecutingError(Exception): + """This Async in not currently executing.""" + + +class AlreadyExecutedError(Exception): + """This Async has already been executed.""" + + +class AlreadyExecutingError(Exception): + """This Async is currently executing.""" + + +class Abort(Exception): + """This Async needs to be aborted immediately. Only an info level logging + message will be output about the aborted job. + """ + + +class AbortAndRestart(Exception): + """This Async needs to be aborted immediately and restarted.""" + + +class AsyncRecursionError(Abort): + """This Async has hit the max recursion depth, it should be aborted.""" + + +class ContextAlreadyStartedError(Exception): + """Attempt to set context on an Async that is already executing in a + context. + """ + + +class ContextExistsError(Exception): + """Call made within context that should not be.""" + + +class CorruptContextError(Exception): + """ExecutionContext raised when the execution context stack is corrupted. + """ + def __init__(self, *exc_info): + self.exc_info = exc_info + + +class AlreadyInContextError(Exception): + """Attempt to set context on an Async that is already executing in a + context. + """ + + +class NotInContextError(Exception): + """Call that requires context made outside context.""" + + + diff --git a/furious/extras/__init__.py b/furious/extras/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/furious/extras/appengine/__init__.py b/furious/extras/appengine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/furious/extras/appengine/ndb_persistence.py b/furious/extras/appengine/ndb_persistence.py new file mode 100644 index 0000000..dcc5ce6 --- /dev/null +++ b/furious/extras/appengine/ndb_persistence.py @@ -0,0 +1,361 @@ +# +# Copyright 2014 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""This module contains the default functions to use when performing +persistence operations backed by the App Engine ndb library. +""" +import json +import logging +import os + +from itertools import imap +from itertools import islice +from itertools import izip + +from random import shuffle + +from google.appengine.ext import ndb + +from furious.context.context import ContextResultBase +from furious import config + +CLEAN_QUEUE = config.get_completion_cleanup_queue() +DEFAULT_QUEUE = config.get_completion_default_queue() +CLEAN_DELAY = config.get_completion_cleanup_delay() +QUEUE_HEADER = 'HTTP_X_APPENGINE_QUEUENAME' + + +class FuriousContextNotFoundError(Exception): + """FuriousContext entity not found in the datastore.""" + + +class FuriousContext(ndb.Model): + """NDB entity to store a Furious Context as JSON.""" + + context = ndb.JsonProperty(indexed=False, compressed=True) + + @classmethod + def from_context(cls, context): + """Create a `cls` entity from a context.""" + return cls(id=context.id, context=context.to_dict()) + + @classmethod + def from_id(cls, id): + """Load a `cls` entity and instantiate the Context it stores.""" + from furious.context import Context + + # TODO: Handle exceptions and retries here. + entity = cls.get_by_id(id) + if not entity: + raise FuriousContextNotFoundError( + "Context entity not found for: {}".format(id)) + + return Context.from_dict(entity.context) + + +class FuriousAsyncMarker(ndb.Model): + """This entity serves as a 'complete' marker.""" + + result = ndb.JsonProperty(indexed=False, compressed=True) + status = ndb.IntegerProperty(indexed=False) + + @property + def success(self): + from furious.async import AsyncResult + return self.status != AsyncResult.ERROR + + +class FuriousCompletionMarker(ndb.Model): + """This entity serves as a 'complete' marker for the entire context.""" + + complete = ndb.BooleanProperty(default=False, indexed=False) + has_errors = ndb.BooleanProperty(default=False, indexed=False) + + +class ContextResult(ContextResultBase): + + BATCH_SIZE = 10 + + def __init__(self, context): + self._context = context + self._task_cache = {} + self._marker = None + + @property + def _tasks(self): + if self._task_cache: + return ((key, task) for key, task in self._task_cache.iteritems()) + + return iter_context_results(self._context, self.BATCH_SIZE, + self._task_cache) + + @property + def _completion_marker(self): + if not self._marker: + self._marker = FuriousCompletionMarker.get_by_id( + self._context.id) + + return self._marker + + def items(self): + """Yield the async reuslts for the context.""" + for key, task in self._tasks: + if not (task and task.result): + yield key, None + else: + yield key, json.loads(task.result)["payload"] + + def values(self): + """Yield the async reuslt values for the context.""" + for _, task in self._tasks: + if not (task and task.result): + yield None + else: + yield json.loads(task.result)["payload"] + + def has_errors(self): + """Return the error flag from the completion marker.""" + if self._completion_marker: + return self._completion_marker.has_errors + + return False + + +def context_completion_checker(async): + """Persist async marker and async the completion check""" + + store_async_marker(async.id, async.result.status if async.result else -1) + + logging.debug("Async check completion for: %s", async.context_id) + current_queue = _get_current_queue() + from furious.async import Async + logging.debug("Completion Check queue:%s", current_queue) + Async(_completion_checker, queue=current_queue, + args=(async.id, async.context_id)).start() + + return True + + +def _get_current_queue(): + + return os.environ.get(QUEUE_HEADER, DEFAULT_QUEUE) + + +def _completion_checker(async_id, context_id): + """Check if all Async jobs within a Context have been run.""" + + if not context_id: + logging.debug("Context for async %s does not exist", async_id) + return + + context = FuriousContext.from_id(context_id) + marker = FuriousCompletionMarker.get_by_id(context_id) + + if marker and marker.complete: + logging.info("Context %s already complete" % context_id) + return True + + task_ids = context.task_ids + if async_id in task_ids: + task_ids.remove(async_id) + + logging.debug("Loaded context.") + logging.debug(task_ids) + + done, has_errors = _check_markers(task_ids) + + if not done: + return False + + _mark_context_complete(marker, context, has_errors) + + return True + + +def _check_markers(task_ids, offset=10): + """Returns a flag for markers being found for the task_ids. If all task ids + have markers True will be returned. Otherwise it will return False as soon + as a None result is hit. + """ + + shuffle(task_ids) + has_errors = False + + for index in xrange(0, len(task_ids), offset): + keys = [ndb.Key(FuriousAsyncMarker, id) + for id in task_ids[index:index + offset]] + + markers = ndb.get_multi(keys) + + if not all(markers): + logging.debug("Not all Async's complete") + return False, None + + # Did any of the aync's fail? Check the success property on the + # AsyncResult. + has_errors = not all((marker.success for marker in markers)) + + return True, has_errors + + +@ndb.transactional +def _mark_context_complete(marker, context, has_errors): + """Transactionally 'complete' the context.""" + + current = None + + if marker: + current = marker.key.get() + + if not current: + return False + + if current and current.complete: + return False + + current.complete = True + current.has_errors = has_errors + current.put() + + # Kick off completion tasks. + _insert_post_complete_tasks(context) + + return True + + +def _insert_post_complete_tasks(context): + """Insert the event's asyncs and cleanup tasks.""" + + logging.debug("Context %s is complete.", context.id) + + # Async event handlers + context.exec_event_handler('complete', transactional=True) + + # Insert cleanup tasks + try: + # TODO: If tracking results we may not want to auto cleanup and instead + # wait until the results have been accessed. + from furious.async import Async + Async(_cleanup_markers, queue=CLEAN_QUEUE, + args=[context.id, context.task_ids], + task_args={'countdown': CLEAN_DELAY}).start() + except: + pass + + +def _cleanup_markers(context_id, task_ids): + """Delete the FuriousAsyncMarker entities corresponding to ids.""" + + logging.debug("Cleanup %d markers for Context %s", + len(task_ids), context_id) + + # TODO: Handle exceptions and retries here. + delete_entities = [ndb.Key(FuriousAsyncMarker, id) for id in task_ids] + delete_entities.append(ndb.Key(FuriousCompletionMarker, context_id)) + + ndb.delete_multi(delete_entities) + + logging.debug("Markers cleaned.") + + +def load_context(id): + """Load a Context object by it's id.""" + + return FuriousContext.from_id(id) + + +def store_context(context): + """Persist a furious.context.Context object to the datastore by loading it + into a FuriousContext ndb.Model. + """ + + logging.debug("Attempting to store Context %s.", context.id) + + entity = FuriousContext.from_context(context) + + # TODO: Handle exceptions and retries here. + marker = FuriousCompletionMarker(id=context.id) + key, _ = ndb.put_multi((entity, marker)) + + logging.debug("Stored Context with key: %s.", key) + + return key + + +def store_async_result(async_id, async_result): + """Persist the Async's result to the datastore.""" + + logging.debug("Storing result for %s", async_id) + + key = FuriousAsyncMarker( + id=async_id, result=json.dumps(async_result.to_dict()), + status=async_result.status).put() + + logging.debug("Setting Async result %s using marker: %s.", async_result, + key) + + +def store_async_marker(async_id, status): + """Persist a marker indicating the Async ran to the datastore.""" + + logging.debug("Attempting to mark Async %s complete.", async_id) + + # QUESTION: Do we trust if the marker had a flag result to just trust it? + marker = FuriousAsyncMarker.get_by_id(async_id) + + if marker: + logging.debug("Marker already exists for %s.", async_id) + return + + # TODO: Handle exceptions and retries here. + key = FuriousAsyncMarker(id=async_id, status=status).put() + + logging.debug("Marked Async complete using marker: %s.", key) + + +def iter_context_results(context, batch_size=10, task_cache=None): + """Yield out the results found on the markers for the context task ids.""" + + for futures in iget_batches(context.task_ids, batch_size=batch_size): + for key, future in futures: + task = future.get_result() + + if task_cache is not None: + task_cache[key.id()] = task + + yield key.id(), task + + +def iget_batches(task_ids, batch_size=10): + """Yield out a map of the keys and futures in batches of the batch size + passed in. + """ + + make_key = lambda _id: ndb.Key(FuriousAsyncMarker, _id) + for keys in i_batch(imap(make_key, task_ids), batch_size): + yield izip(keys, ndb.get_multi_async(keys)) + + +def i_batch(items, size): + """Generator that iteratively batches items to a max size and consumes the + items as each batch is yielded. + """ + for items_batch in iter(lambda: tuple(islice(items, size)), + tuple()): + yield items_batch + + +def get_context_result(context): + return ContextResult(context) diff --git a/furious/extras/insert_task_handlers.py b/furious/extras/insert_task_handlers.py new file mode 100644 index 0000000..ae81fe8 --- /dev/null +++ b/furious/extras/insert_task_handlers.py @@ -0,0 +1,32 @@ +from furious.context.context import _insert_tasks +from furious.context.context import _tasks_to_reinsert + + +def insert_tasks_ignore_duplicate_names(tasks, queue, *args, **kwargs): + """Insert a batch of tasks into a specific queue. If a + DuplicateTaskNameError is raised, loop through the tasks and insert the + remaining, ignoring and logging the duplicate tasks. + + Returns the number of successfully inserted tasks. + """ + + from google.appengine.api import taskqueue + + try: + inserted = _insert_tasks(tasks, queue, *args, **kwargs) + + return inserted + except taskqueue.DuplicateTaskNameError: + # At least one task failed in our batch, attempt to re-insert the + # remaining tasks. Named tasks can never be transactional. + reinsert = _tasks_to_reinsert(tasks, transactional=False) + + count = len(reinsert) + inserted = len(tasks) - count + + # Our subsequent task inserts should raise TaskAlreadyExistsError at + # least once, but that will be swallowed by _insert_tasks. + for task in reinsert: + inserted += _insert_tasks([task], queue, *args, **kwargs) + + return inserted diff --git a/furious/extras/xsrf.py b/furious/extras/xsrf.py new file mode 100644 index 0000000..032656e --- /dev/null +++ b/furious/extras/xsrf.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +""" + webapp2_extras.xsrf + =================== + + Helpers for defending against cross-site request forgery attacks. + + :copyright: 2012 by tipfy.org. + :license: Apache Sotware License, see LICENSE for details. +""" + +__author__ = 'John Lockwood' + +import base64 +import hmac +import hashlib +import time + +class XSRFException(Exception): + pass + +class XSRFTokenMalformed(XSRFException): + pass + +class XSRFTokenExpiredException(XSRFException): + pass + +class XSRFTokenInvalid(XSRFException): + pass + +class XSRFToken(object): + _DELIMITER = '|' + + def __init__(self, user_id, secret, current_time=None): + """Initializes the XSRFToken object. + + :param user_id: + A string representing the user that the token will be valid for. + :param secret: + A string containing a secret key that will be used to seed the + hash used by the :class:`XSRFToken`. + :param current_time: + An int representing the number of seconds since the epoch. Will be + used by `verify_token_string` to check for token expiry. If `None` + then the current time will be used. + """ + self.user_id = user_id + self.secret = secret + if current_time is None: + self.current_time = int(time.time()) + else: + self.current_time = int(current_time) + + def _digest_maker(self): + return hmac.new(self.secret, digestmod=hashlib.sha1) + + def generate_token_string(self, action=None): + """Generate a hash of the given token contents that can be verified. + + :param action: + A string representing the action that the generated hash is valid + for. This string is usually a URL. + :returns: + A string containing the hash contents of the given `action` and the + contents of the `XSRFToken`. Can be verified with + `verify_token_string`. The string is base64 encoded so it is safe + to use in HTML forms without escaping. + """ + digest_maker = self._digest_maker() + digest_maker.update(self.user_id) + digest_maker.update(self._DELIMITER) + if action: + digest_maker.update(action) + digest_maker.update(self._DELIMITER) + + digest_maker.update(str(self.current_time)) + return base64.urlsafe_b64encode( + self._DELIMITER.join([digest_maker.hexdigest(), + str(self.current_time)])) + + def verify_token_string(self, + token_string, + action=None, + timeout=None, + current_time=None): + """Generate a hash of the given token contents that can be verified. + + :param token_string: + A string containing the hashed token (generated by + `generate_token_string`). + :param action: + A string containing the action that is being verified. + :param timeout: + An int or float representing the number of seconds that the token + is valid for. If None then tokens are valid forever. + :current_time: + An int representing the number of seconds since the epoch. Will be + used by to check for token expiry if `timeout` is set. If `None` + then the current time will be used. + :raises: + XSRFTokenMalformed if the given token_string cannot be parsed. + XSRFTokenExpiredException if the given token string is expired. + XSRFTokenInvalid if the given token string does not match the + contents of the `XSRFToken`. + """ + try: + decoded_token_string = base64.urlsafe_b64decode(token_string) + except TypeError: + raise XSRFTokenMalformed() + + split_token = decoded_token_string.split(self._DELIMITER) + if len(split_token) != 2: + raise XSRFTokenMalformed() + + try: + token_time = int(split_token[1]) + except ValueError: + raise XSRFTokenMalformed() + + if timeout is not None: + if current_time is None: + current_time = time.time() + # If an attacker modifies the plain text time then it will not match + # the hashed time so this check is sufficient. + if (token_time + timeout) < current_time: + raise XSRFTokenExpiredException() + + expected_token = XSRFToken(self.user_id, self.secret, token_time) + expected_token_string = expected_token.generate_token_string(action) + + if len(expected_token_string) != len(token_string): + raise XSRFTokenInvalid() + + # Compare the two strings in constant time to prevent timing attacks. + different = 0 + for a, b in zip(token_string, expected_token_string): + different |= ord(a) ^ ord(b) + if different: + raise XSRFTokenInvalid() diff --git a/furious/handlers/__init__.py b/furious/handlers/__init__.py index 7e3c88b..b153283 100644 --- a/furious/handlers/__init__.py +++ b/furious/handlers/__init__.py @@ -15,18 +15,22 @@ # import json +import time import logging -from ..async import Async -from .. import context -from ..processors import run_job +from furious.async import async_from_options +from furious import context +from furious.processors import run_job def process_async_task(headers, request_body): """Process an Async task and execute the requested function.""" async_options = json.loads(request_body) - async = Async.from_dict(async_options) + async = async_from_options(async_options) + + _log_task_info(headers, + extra_task_info=async.get_options().get('_extra_task_info')) logging.info(async._function_path) @@ -36,4 +40,19 @@ def process_async_task(headers, request_body): return 200, async._function_path +def _log_task_info(headers, extra_task_info=None): + """Processes the header from task requests to log analytical data.""" + ran_at = time.time() + task_eta = float(headers.get('X-Appengine-Tasketa', 0.0)) + task_info = { + 'retry_count': headers.get('X-Appengine-Taskretrycount', ''), + 'execution_count': headers.get('X-Appengine-Taskexecutioncount', ''), + 'task_eta': task_eta, + 'ran': ran_at, + 'gae_latency_seconds': ran_at - task_eta + } + + if extra_task_info: + task_info['extra'] = extra_task_info + logging.debug('TASK-INFO: %s', json.dumps(task_info)) diff --git a/furious/handlers/webapp.py b/furious/handlers/webapp.py index e5e3fdb..9ae3c8e 100644 --- a/furious/handlers/webapp.py +++ b/furious/handlers/webapp.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import webapp2 -from . import process_async_task - +from furious.handlers import process_async_task +from furious.errors import AbortAndRestart +from furious.config import get_csrf_check class AsyncJobHandler(webapp2.RequestHandler): """Handles requests for the webapp framework.""" @@ -29,11 +29,22 @@ def post(self): def _handle_task(self): """Pass request info to the async framework.""" - headers = self.request.headers + # Check for CSRF + get_csrf_check()(self.request) - staus_code, output = process_async_task(headers, self.request.body) + headers = self.request.headers - self.response.set_status(staus_code) + message = None + try: + status_code, output = process_async_task( + headers, self.request.body) + except AbortAndRestart as restart: + # Async retry status code + status_code = 549 + message = 'Retry Async Task' + output = str(restart) + + self.response.set_status(status_code, message) self.response.out.write(output) app = webapp2.WSGIApplication([ diff --git a/furious/job_utils.py b/furious/job_utils.py index ffd7dec..5ce1572 100644 --- a/furious/job_utils.py +++ b/furious/job_utils.py @@ -18,11 +18,7 @@ Functions to help with encoding and decoding job information. """ -import sys - - -class BadFunctionPathError(Exception): - """Invalid function path.""" +from furious import errors def get_function_path_and_options(function): @@ -40,68 +36,128 @@ def get_function_path_and_options(function): """ # Try to pop the options off whatever they passed in. options = getattr(function, '_async_options', None) + return reference_to_path(function), options - if isinstance(function, basestring): - # This is a function name in str form. - import re - if not re.match(r'^[^\d\W]([a-zA-Z._]|((? 0) + + # Keep processing if we have processed any tasks and are under our limit. + while processed: + + processed = _run(taskq_service, queue_names, non_furious_url_prefixes, + non_furious_handler, enable_retries) + tasks_processed += processed + iterations += 1 + + if max_iterations and iterations >= max_iterations: + break + + return {'iterations': iterations, 'tasks_processed': tasks_processed} + + +def get_tasks(taskq_service, queue_names=None): + """ + Get all tasks from queues and return them in a dict keyed by queue_name. + If queue_names not specified, returns tasks from all queues. + + :param taskq_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param queue_names: :class: `list` of queue name strings. + """ + + # Make sure queue_names is a list + if isinstance(queue_names, basestring): + queue_names = [queue_names] + + if not queue_names: + queue_names = get_queue_names(taskq_service) + + task_dict = defaultdict(list) + + for queue_name in queue_names: + # Get tasks + tasks = taskq_service.GetTasks(queue_name) + + task_dict[queue_name].extend(tasks) + + return task_dict + + +def add_tasks(taskq_service, task_dict): + """ + Allow readding of multiple tasks across multiple queues. + The task_dict is a dictionary with tasks for each queue, keyed by queue + name. + Tasks themselves can be dicts like those received from GetTasks() or Task + instances. + + :param taskq_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param queue_names: :class: `dict` of queue name: tasks dictionary. + """ + + num_added = 0 + + # Get the descriptions so we know when to specify PULL mode. + queue_descriptions = taskq_service.GetQueues() + queue_desc_dict = dict((queue_desc['name'], queue_desc) + for queue_desc in queue_descriptions) + + # Loop over queues and add tasks for each. + for queue_name, tasks in task_dict.items(): + + queue = taskqueue.Queue(queue_name) + + is_pullqueue = ('pull' == queue_desc_dict[queue_name]['mode']) + + tasks_to_add = [] + + # Ensure tasks are formatted to add to queues. + for task in tasks: + + # If already formatted as a Task, add it. + if isinstance(task, taskqueue.Task): + tasks_to_add.append(task) + continue + + # If in dict format that comes from GetTasks(), format it as a Task + # First look for payload. If no payload, look for body to decode. + if 'payload' in task: + payload = task['payload'] + else: + payload = base64.b64decode(task.get('body')) + + # Setup different parameters for pull and push queues + if is_pullqueue: + task_obj = taskqueue.Task(payload=payload, + name=task.get('name'), + method='PULL', + url=task.get('url')) + else: + task_obj = taskqueue.Task(payload=payload, + name=task.get('name'), + method=task.get('method')) + + tasks_to_add.append(task_obj) + + # Add tasks to queue + if tasks_to_add: + queue.add(tasks_to_add) + num_added += len(tasks_to_add) + + return num_added + + +def purge_tasks(taskq_service, queue_names=None): + """Remove all tasks from queues. + + :param taskq_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param queue_names: :class: `list` of queue name strings. + """ + + # Make sure queue_names is a list + if isinstance(queue_names, basestring): + queue_names = [queue_names] + + if not queue_names: + queue_names = get_queue_names(taskq_service) + + num_tasks = 0 + + for queue_name in queue_names: + # Get tasks to help give some feedback + tasks = taskq_service.GetTasks(queue_name) + num_tasks += len(tasks) + + taskq_service.FlushQueue(queue_name) + + return num_tasks + + +def get_queue_names(taskq_service, mode=None): + """Returns push queue names from the Task Queue service. + + :param taskq_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param mode: :class: `str` Task queue mode "pull" or "push" + """ + + queue_descriptions = taskq_service.GetQueues() + + return [description['name'] + for description in queue_descriptions] + + +def get_pull_queue_names(taskq_service, mode=None): + """Returns pull queue names from the Task Queue service. + + :param taskq_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param mode: :class: `str` Task queue mode "pull" or "push" + """ + + queue_descriptions = taskq_service.GetQueues() + + return [description['name'] + for description in queue_descriptions + if 'pull' == description.get('mode')] + + +def get_push_queue_names(taskq_service, mode=None): + """Returns push queue names from the Task Queue service. + + :param taskq_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param mode: :class: `str` Task queue mode "pull" or "push" + """ + + queue_descriptions = taskq_service.GetQueues() + + return [description['name'] + for description in queue_descriptions + if 'push' == description.get('mode')] + + +class Runner(object): + """A class to help run pull queues. + + Allows parameters such as taskq_service and queue_names be specified at + __init__ instead of in each run() call. + """ + # TODO: WRITE UNIT TESTS FOR ME. + + def __init__(self, taskq_service, queue_names=None): + """Store taskq_service and optionally queue_name list for reuse.""" + + self.taskq_service = taskq_service + + if None == queue_names: + self.queue_names = get_push_queue_names(self.taskq_service) + else: + self.queue_names = queue_names + + def run(self, max_iterations=None): + """Run the existing tasks for all pushqueue.""" + + return run(self.taskq_service, self.queue_names, max_iterations) + + def run_queue(self, queue_name): + """Run all the existing tasks for one queue.""" + + return run_queue(self.taskq_service, queue_name) + + +def _execute_wrapped_task(task_dict, queue_service, queue_name, + non_furious_url_prefixes=None, + non_furious_handler=None): + """ + Wrap _execute_task in a try/except. + + Also, set number of retries in the environment. + Return True if task executes normally, False if it fails. + """ + + # Set retry count in environment. + header_dict = dict(task_dict['headers']) + os.environ['HTTP_X_APPENGINE_TASKRETRYCOUNT'] = header_dict.get( + 'X-AppEngine-TaskRetryCount', '0') + + success = False + try: + _execute_task( + task_dict, non_furious_url_prefixes=non_furious_url_prefixes, + non_furious_handler=non_furious_handler) + success = True + + except Exception, e: + # Tests for Abort don't allow logging.exception here, so only log as a + # warning and include the traceback. + stack = traceback.format_exc() + logging.warning("%s\n%s" % (repr(e), repr(stack))) + + del os.environ['HTTP_X_APPENGINE_TASKRETRYCOUNT'] + + return success + + +def _execute_task(task, non_furious_url_prefixes=None, + non_furious_handler=None): + """Extract the body and header from the task and process it. + + :param task: :class: `taskqueue.Task` + :param non_furious_url_prefixes: :class: `list` of url prefixes that the + furious task runner will run. + :param non_furious_handler: :class: `func` handler for non furious tasks to + run within. + """ + if not _is_furious_task(task, non_furious_url_prefixes, + non_furious_handler): + return + + # Ensure each test looks like it is in a new request. + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + # Decode the body and process the task. + body = base64.b64decode(task['body']) + + return_code, func_path = process_async_task(dict(task['headers']), body) + + # TODO: Possibly do more with return_codes. + + # Cleanup context since we will be executing more tasks in this process. + _clear_context() + + del os.environ['REQUEST_ID_HASH'] + + +def _is_furious_task(task, non_furious_url_prefixes=None, + non_furious_handler=None): + """Return flag if task is a Furious task. If no non_furious_url_prefixes is + passed in the task will be flagged as Furious. Otherwise it will compare + the task url to the furious_url_prefix. If not a Furious task it will try + to run it within the non_furious_handler if one is handed in. + + :param task: :class: `taskqueue.Task` + :param non_furious_url_prefixes: :class: `list` of url prefixes that the + furious task runner will run. + :param non_furious_handler: :class: `func` handler for non furious tasks to + run within. + """ + if not non_furious_url_prefixes: + return True + + task_url = task['url'] + + for non_furious_url_prefix in non_furious_url_prefixes: + if task_url.startswith(non_furious_url_prefix): + if non_furious_handler: + logging.info("Passing %s to non Furious handler %s", task, + non_furious_handler) + non_furious_handler(task) + + return False + + return True + + +def _run(taskq_service, queue_names, non_furious_url_prefixes=None, + non_furious_handler=None, enable_retries=False): + """Run individual tasks in push queues. + + :param taskq_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param queue_names: :class: `list` of queue name strings + :param non_furious_url_prefixes: :class: `list` of url prefixes that the + furious task runner will run. + :param non_furious_handler: :class: `func` handler for non furious tasks to + run within. + :type enable_retries: bool Whether to enable task retries. + """ + + num_processed = 0 + + # Process each queue + # TODO: Round robin instead of one queue at a time. + for queue_name in queue_names: + num_processed += run_queue(taskq_service, queue_name, + non_furious_url_prefixes, + non_furious_handler, enable_retries) + + return num_processed + + +def run_random(queue_service, queues, random_seed=123, max_tasks=100): + """Run individual tasks in push queues randomly. This will run by + randomly picking a queue and then randomly picking a task from that queue. + Once the task is ran we pick a different random queue and random task + until we've either exhausted the queues or we have ran the 'max_tasks'. + + This will only run tasks from 'push' queues. + + :param queue_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param queues: :class: `dict` of queue 'descriptions' where each + description is a 'dict' with at least 'mode' and 'name'. + :param random_seed: :class: 'int' to be used to seed random + :param max_tasks: :class: 'int' to indicate the max number of tasks + to be ran before we exit. + :returns: :class: 'int' with count of tasks ran + """ + if not queues: + return 0 + + queue_count = len(queues) + + random.seed(random_seed) + + # Run until we hit the task limit + num_processed = 0 + while num_processed < max_tasks: + + # Start by grabbing a random queue to run from + queue_index = random.randrange(queue_count) + + # Grab the queue description + queue_desc = queues[queue_index] + processed_queue_count = 0 + + # Keep trying to run a task until we either successfully run or + # we have ran through all of the queues. + task_ran = False + while not task_ran and processed_queue_count < queue_count: + + # Only process from push queues. + if queue_desc.get('mode') == 'push': + queue_name = queue_desc['name'] + task_ran = _run_random_task_from_queue(queue_service, queue_name) + + if task_ran: + num_processed += 1 + else: + # There isn't a task for the queue so we need to check the next + # queue in the list. + queue_index += 1 + queue_desc = queues[queue_index % queue_count] + processed_queue_count += 1 + + # If we ran through all of the queues without running a task then + # we can break out + if not task_ran: + break + + return num_processed + + +def _run_random_task_from_queue(queue_service, queue_name): + """Attempts to run a random task from the queue identified + by queue_name. Returns True if a task was ran otherwise + returns False. + + :param queue_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param queue_name: :class: `basestring` with name of queue to run from. + :returns: :class: 'bool' true if we ran a task; false otherwise + """ + task = _fetch_random_task_from_queue(queue_service, queue_name) + + if task and task.get('name'): + _execute_task(task) + + queue_service.DeleteTask(queue_name, task.get('name')) + + return True + + return False + + +def _fetch_random_task_from_queue(queue_service, queue_name): + """Returns a random task description from the queue identified by queue_name + if there exists at least one task in the queue. + + :param queue_service: :class: `taskqueue_stub.TaskQueueServiceStub` + :param queue_name: :class: `basestring` with name of queue to pull from. + :returns: :class: 'dict' containing the task description to run. + """ + tasks = queue_service.GetTasks(queue_name) + + if not tasks: + return None + + task = random.choice(tasks) + + if not task: + return None + + return task + + +### Deprecated ### + +def execute_queues(queues, queue_service): + """ DEPRECATED + Remove this as soon as references to this in other libraries are gone. + Use run() or Runner.run() instead of this. + + Run individual tasks in push queues. + """ + import logging + logging.warning('This method is deprecated, switch to ') + + num_processed = False + + # Process each queues + for queue_desc in queues: + + # Don't pull anything from pull queues. + if queue_desc.get('mode') == 'pull': + continue + + num_processed = (run_queue(queue_service, queue_desc['name']) + or num_processed) + + return bool(num_processed) diff --git a/furious/tests/context/test_auto_context.py b/furious/tests/context/test_auto_context.py new file mode 100644 index 0000000..314658c --- /dev/null +++ b/furious/tests/context/test_auto_context.py @@ -0,0 +1,128 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest + +from google.appengine.ext import testbed + +from mock import Mock +from mock import patch + + +class TestAutoContextTestCase(unittest.TestCase): + """Test the AutoContext class.""" + + def setUp(self): + """Setup the test harness and a request hash.""" + import os + import uuid + + self.harness = testbed.Testbed() + self.harness.activate() + self.harness.init_taskqueue_stub() + + # Backup environment + self._orig_environ = os.environ.copy() + # Ensure each test looks like it is in a new request. + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + def tearDown(self): + """Deactive the test harness, and restore the environment.""" + import os + + self.harness.deactivate() + os.environ.clear() + os.environ.update(self._orig_environ) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_add_job_to_context_multiple_batches(self, queue_add_mock): + """Ensure adding more tasks than the batch_size causes multiple batches + to get inserted. + + Adding 3 asyncs with a batch_size of 2 should result in two queue.adds, + containing 2 and 1 task respectively. + + Also ensure that the remaining task is inserted upon exiting the + context. + """ + from furious.async import Async + from furious.context.auto_context import AutoContext + + batch_size = 2 + + with AutoContext(batch_size) as ctx: + # First batch + job1 = ctx.add('test', args=[1, 2]) + job2 = ctx.add('test2', args=[1, 2]) + + # Second batch (not added until "with" section ends)) + job3 = ctx.add('test3', args=[1, 2]) + + # Ensure the first two jobs were inserted in the first batch. + self.assertIsInstance(job1, Async) + self.assertIsInstance(job2, Async) + self.assertEqual(1, queue_add_mock.call_count) + #Ensure only two tasks were inserted + tasks_added = queue_add_mock.call_args[0][0] + self.assertEqual(2, len(tasks_added)) + + # Ensure the third job was inserted when the context exited. + self.assertIsInstance(job3, Async) + # Ensure add has now been called twice. + self.assertEqual(2, queue_add_mock.call_count) + # Ensure only one task was inserted + tasks_added = queue_add_mock.call_args[0][0] + self.assertEqual(1, len(tasks_added)) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_add_job_to_context_batch_size_unspecified(self, queue_add_mock): + """When batch_size is None or 0, the default behavior of Context is + used. All the tasks are added to the queue when the context is exited. + """ + from furious.async import Async + from furious.context.auto_context import AutoContext + + with AutoContext() as ctx: + + job1 = ctx.add('test', args=[1, 2]) + job2 = ctx.add('test2', args=[1, 2]) + job3 = ctx.add('test3', args=[1, 2]) + + self.assertIsInstance(job1, Async) + self.assertIsInstance(job2, Async) + self.assertIsInstance(job3, Async) + + # Ensure no batches of tasks are inserted yet. + self.assertFalse(queue_add_mock.called) + + # Ensure the list of tasks added when the context exited. + self.assertEqual(1, queue_add_mock.call_count) + # Ensure the three tasks were inserted + tasks_added = queue_add_mock.call_args[0][0] + self.assertEqual(3, len(tasks_added)) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_no_jobs(self, queue_add_mock): + """When no Asyncs are added to the context, ensure that there are no + errors and nothing is added to the task queue. + """ + from furious.context.auto_context import AutoContext + + with AutoContext(3): + pass + + # Ensure queue.add() was never called. + self.assertEqual(0, queue_add_mock.call_count) diff --git a/furious/tests/context/test_context.py b/furious/tests/context/test_context.py index 3388a69..8fcf6f2 100644 --- a/furious/tests/context/test_context.py +++ b/furious/tests/context/test_context.py @@ -18,6 +18,8 @@ from google.appengine.ext import testbed +from mock import call +from mock import Mock from mock import patch @@ -31,6 +33,19 @@ def test_new(self): self.assertIsInstance(new(), Context) + def test_new_auto_context(self): + """Ensure new returns a new AutoContext when batch size is specified. + """ + from furious.context import AutoContext + from furious.context import new + + batch_size = 100 + + context = new(batch_size=batch_size) + + self.assertIsInstance(context, AutoContext) + self.assertEqual(context.batch_size, batch_size) + def test_new_adds_to_registry(self): """Ensure new adds new contexts to the context registry.""" from furious.context import Context @@ -63,6 +78,60 @@ def test_context_works(self): with Context(): pass + def test_context_requires_insert_tasks(self): + """Ensure Contexts require a callable insert_tasks function.""" + from furious.context import Context + + self.assertRaises(TypeError, Context, insert_tasks='nope') + + def test_context_gets_id(self): + """Ensure a new Context gets an id generated.""" + from furious.context import Context + + self.assertTrue(Context().id) + + @patch('uuid.uuid4', autospec=True) + def test_id_added_to_options(self, uuid_patch): + """Ensure random context id gets added to options.""" + from furious.context import Context + + id = 'random-id' + uuid_patch.return_value.hex = id + + context = Context() + + self.assertEqual(context.id, id) + self.assertEqual(context._options['id'], id) + + def test_context_gets_one_id(self): + """Ensure a new Context gets an id only generated once.""" + from furious.context import Context + + context = Context() + + id1 = context.id + id2 = context.id + self.assertEqual(id1, id2) + self.assertEqual(context.id, id1) + + def test_context_gets_assigned_id(self): + """Ensure a new Context keeps its assigned id.""" + from furious.context import Context + + self.assertEqual('test_id_weee', Context(id='test_id_weee').id) + + def test_insert_success(self): + """Ensure a new Context has an insert_success of 0.""" + from furious.context import Context + + self.assertEqual(0, Context().insert_success) + + def test_insert_failed(self): + """Ensure a new Context has an insert_failed of 0.""" + from furious.context import Context + + self.assertEqual(0, Context().insert_failed) + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) def test_add_job_to_context_works(self, queue_add_mock): """Ensure adding a job works.""" @@ -73,7 +142,8 @@ def test_add_job_to_context_works(self, queue_add_mock): job = ctx.add('test', args=[1, 2]) self.assertIsInstance(job, Async) - queue_add_mock.assert_called_once() + self.assertEqual(1, ctx.insert_success) + self.assertEqual(1, queue_add_mock.call_count) @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) def test_bubbling_exceptions(self, queue_add_mock): @@ -104,6 +174,8 @@ def test_nested_context_works(self, queue_add_mock): self.assertIsInstance(job, Async) self.assertIsInstance(job2, Async) + self.assertEqual(1, ctx.insert_success) + self.assertEqual(1, ctx2.insert_success) self.assertEqual(2, queue_add_mock.call_count) @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) @@ -115,8 +187,24 @@ def test_add_multiple_jobs_to_context_works(self, queue_add_mock): for _ in range(10): ctx.add('test', args=[1, 2]) - queue_add_mock.assert_called_once() + self.assertEqual(1, queue_add_mock.call_count) self.assertEqual(10, len(queue_add_mock.call_args[0][0])) + self.assertEqual(10, ctx.insert_success) + + @patch('google.appengine.api.taskqueue.Queue', auto_spec=True) + def test_added_asyncs_get_context_id(self, queue_mock): + """Ensure Asyncs added to context get context id.""" + from furious.async import Async + from furious.context import Context + + asyncs = [Async('test', id=i) for i in xrange(100, 110)] + + with Context() as ctx: + for async in asyncs: + ctx.add(async) + self.assertEqual(ctx.id, async.get_options()['_context_id']) + + self.assertEqual(10, ctx.insert_success) @patch('google.appengine.api.taskqueue.Queue', auto_spec=True) def test_added_to_correct_queue(self, queue_mock): @@ -128,6 +216,7 @@ def test_added_to_correct_queue(self, queue_mock): ctx.add('test', args=[1, 2], queue='A') queue_mock.assert_called_once_with(name='A') + self.assertEqual(2, ctx.insert_success) def test_add_jobs_to_multiple_queues(self): """Ensure adding jobs to multiple queues works as expected.""" @@ -156,6 +245,209 @@ def add(self, *args, **kwargs): self.assertEqual(2, len(queue_registry['A']._calls[0][0][0])) self.assertEqual(1, len(queue_registry['B']._calls[0][0][0])) self.assertEqual(1, len(queue_registry['C']._calls[0][0][0])) + self.assertEqual(4, ctx.insert_success) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_add_task_fails(self, queue_add_mock): + """Ensure insert_failed and insert_success are calculated correctly.""" + from google.appengine.api.taskqueue import TaskAlreadyExistsError + from furious.context import Context + + def queue_add(tasks, transactional=False): + if len(tasks) != 2: + raise TaskAlreadyExistsError() + + queue_add_mock.side_effect = queue_add + + with Context() as ctx: + ctx.add('test', args=[1, 2], queue='A') + ctx.add('test', args=[1, 2], queue='B') + ctx.add('test', args=[1, 2], queue='B') + + self.assertEqual(2, ctx.insert_success) + self.assertEqual(1, ctx.insert_failed) + + def test_to_dict(self): + """Ensure to_dict returns a dictionary representation of the Context. + """ + import copy + + from furious.context import Context + + options = { + 'persistence_engine': 'persistence_engine', + 'unkown': True, + 'id': 'anid' + } + + context = Context(**copy.deepcopy(options)) + + # This stuff gets dumped out by to_dict(). + options.update({ + 'insert_tasks': 'furious.context.context._insert_tasks', + '_tasks_inserted': False, + '_task_ids': [], + 'id': 'anid' + }) + + self.assertEqual(options, context.to_dict()) + + def test_to_dict_with_callbacks(self): + """Ensure to_dict correctly encodes callbacks.""" + import copy + + from furious.async import Async + from furious.context import Context + + options = { + 'id': 'someid', + 'context_id': 'contextid', + 'parent_id': 'parentid', + 'persistence_engine': 'persistence_engine', + 'callbacks': { + 'success': self.__class__.test_to_dict_with_callbacks, + 'failure': "failure_function", + 'exec': Async(target=dir, id='blargh', context_id='contextid', + parent_id='parentid') + } + } + + context = Context(**copy.deepcopy(options)) + + # This stuff gets dumped out by to_dict(). + options.update({ + 'id': 'someid', + 'insert_tasks': 'furious.context.context._insert_tasks', + 'persistence_engine': 'persistence_engine', + '_tasks_inserted': False, + '_task_ids': [], + 'callbacks': { + 'success': ("furious.tests.context.test_context." + "TestContext.test_to_dict_with_callbacks"), + 'failure': "failure_function", + 'exec': {'job': ('dir', None, None), + 'id': 'blargh', + 'context_id': 'contextid', + 'parent_id': 'parentid', + '_recursion': {'current': 0, 'max': 100}, + '_type': 'furious.async.Async'} + } + }) + + self.assertEqual(options, context.to_dict()) + + def test_from_dict(self): + """Ensure from_dict returns the correct Context object.""" + from furious.context import Context + + from furious.context.context import _insert_tasks + + # TODO: persistence_engine needs set to a real persistence module. + + options = { + 'id': 123456, + 'insert_tasks': 'furious.context.context._insert_tasks', + 'random_option': 'avalue', + '_tasks_inserted': True, + '_task_ids': [1, 2, 3, 4], + 'persistence_engine': 'furious.context.context.Context' + } + + context = Context.from_dict(options) + + self.assertEqual(123456, context.id) + self.assertEqual([1, 2, 3, 4], context.task_ids) + self.assertEqual(True, context._tasks_inserted) + self.assertEqual('avalue', context._options.get('random_option')) + self.assertEqual(_insert_tasks, context._insert_tasks) + self.assertEqual(Context, context._persistence_engine) + + def test_from_dict_with_callbacks(self): + """Ensure from_dict reconstructs the Context callbacks correctly.""" + from furious.context import Context + + callbacks = { + 'success': ("furious.tests.context.test_context." + "TestContext.test_to_dict_with_callbacks"), + 'failure': "dir", + 'exec': {'job': ('id', None, None), 'id': 'myid', + 'context_id': 'contextid', + 'parent_id': 'parentid'} + } + + context = Context.from_dict({'callbacks': callbacks}) + + check_callbacks = { + 'success': TestContext.test_to_dict_with_callbacks, + 'failure': dir + } + + callbacks = context._options.get('callbacks') + exec_callback = callbacks.pop('exec') + + correct_dict = {'job': ('id', None, None), + 'parent_id': 'parentid', + 'id': 'myid', + 'context_id': 'contextid', + '_recursion': {'current': 0, 'max': 100}, + '_type': 'furious.async.Async'} + + self.assertEqual(check_callbacks, callbacks) + self.assertEqual(correct_dict, exec_callback.to_dict()) + + def test_reconstitution(self): + """Ensure to_dict(job.from_dict()) returns the same thing.""" + from furious.context import Context + + options = { + 'id': 123098, + 'insert_tasks': 'furious.context.context._insert_tasks', + 'context_id': 'contextid', + 'persistence_engine': + 'furious.job_utils.get_function_path_and_options', + '_tasks_inserted': True, + '_task_ids': [] + } + + context = Context.from_dict(options) + + self.assertEqual(options, context.to_dict()) + + def test_persist_with_no_engine(self): + """Calling persist with no engine should blow up.""" + from furious.context import Context + + context = Context() + self.assertRaises(RuntimeError, context.persist) + + def test_persist_persists(self): + """Calling persist with an engine persists the Context.""" + from furious.context import Context + + persistence_engine = Mock() + persistence_engine.func_name = 'persistence_engine' + persistence_engine.im_class.__name__ = 'engine' + + context = Context(persistence_engine=persistence_engine) + + context.persist() + + persistence_engine.store_context.assert_called_once_with(context) + + def test_load_context(self): + """Calling load with an engine attempts to load the Context.""" + from furious.context import Context + + persistence_engine = Mock() + persistence_engine.func_name = 'persistence_engine' + persistence_engine.im_class.__name__ = 'engine' + persistence_engine.load_context.return_value = Context.from_dict( + {'id': 'ABC123'}) + + context = Context.load('ABC123', persistence_engine) + + persistence_engine.load_context.assert_called_once_with('ABC123') + self.assertEqual('ABC123', context.id) class TestInsertTasks(unittest.TestCase): @@ -169,50 +461,352 @@ def test_no_tasks_doesnt_blow_up(self): """Ensure calling with an empty list doesn't blow up.""" from furious.context.context import _insert_tasks - _insert_tasks((), 'A') + inserted = _insert_tasks((), 'A') + + self.assertEqual(0, inserted) @patch('google.appengine.api.taskqueue.Queue', auto_spec=True) def test_queue_name_is_honored(self, queue_mock): """Ensure the Queue is instantiated with the name.""" from furious.context.context import _insert_tasks - _insert_tasks((None,), 'AbCd') + inserted = _insert_tasks((None,), 'AbCd') queue_mock.assert_called_once_with(name='AbCd') + self.assertEqual(1, inserted) @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) def test_tasks_are_passed_along(self, queue_add_mock): """Ensure the list of tasks are passed along.""" from furious.context.context import _insert_tasks - _insert_tasks(('A', 1, 'B', 'joe'), 'AbCd') + inserted = _insert_tasks(('A', 1, 'B', 'joe'), 'AbCd') queue_add_mock.assert_called_once_with(('A', 1, 'B', 'joe'), transactional=False) + self.assertEqual(4, inserted) + + @patch('time.sleep') + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_task_add_error_TransientError(self, queue_add_mock, mock_sleep): + """Ensure a TransientError gets raised from add if we've specified not + to retry those errors.""" + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + queue_add_mock.side_effect = taskqueue.TransientError + + self.assertRaises( + taskqueue.TransientError, + _insert_tasks, ('A',), 'AbCd', retry_transient_errors=False + ) + + queue_add_mock.assert_called_once_with(('A',), transactional=False) + + @patch('time.sleep') + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_task_add_error_TransientError_with_delay(self, queue_add_mock, + mock_sleep): + """Ensure a TransientError gets retried with the retry_delay that we + have specified.""" + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + queue_add_mock.side_effect = taskqueue.TransientError + + tasks = [taskqueue.Task('A')] + self.assertRaises( + taskqueue.TransientError, + _insert_tasks, tasks, 'AbCd', retry_transient_errors=True, + retry_delay=12 + ) + + self.assertEqual(queue_add_mock.call_count, 2) + mock_sleep.assert_called_once_with(12) + + @patch('time.sleep') + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_task_add_error_TransientError_transactional(self, queue_add_mock, + mock_sleep): + """Ensure a TransientError gets re-raised when transactional=True""" + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + queue_add_mock.side_effect = taskqueue.TransientError + + tasks = [taskqueue.Task('A')] + self.assertRaises( + taskqueue.TransientError, + _insert_tasks, tasks, 'AbCd', + transactional=True, + retry_transient_errors=True, + retry_delay=1, + ) + + self.assertEqual(queue_add_mock.call_count, 1) + self.assertEqual(mock_sleep.call_count, 0) + + @patch('time.sleep') + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_batches_get_split_TransientError(self, queue_add_mock, mock_sleep): + """Ensure TransientErrors retries once, and correctly returns the number + of inserted tasks.""" + + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + queue_add_mock.side_effect = (taskqueue.TransientError, None) + tasks = (taskqueue.Task('A'), taskqueue.Task('1'), taskqueue.Task('B')) + + inserted = _insert_tasks(tasks, 'AbCd') + self.assertEqual(2, queue_add_mock.call_count) + self.assertEqual(3, inserted) + self.assertEqual(mock_sleep.call_count, 1) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_task_add_error_BadTaskStateError(self, queue_add_mock): + """Ensure a BadTaskStateError doesn't get raised from add.""" + from furious.context.context import _insert_tasks + + def raise_error(*args, **kwargs): + from google.appengine.api import taskqueue + raise taskqueue.BadTaskStateError() + + queue_add_mock.side_effect = raise_error + + inserted = _insert_tasks(('A',), 'AbCd') + queue_add_mock.assert_called_once_with(('A',), transactional=False) + self.assertEqual(0, inserted) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_batches_get_split_BadTaskStateError(self, queue_add_mock): + """Ensure a batches get split and retried on BadTaskStateErrors.""" + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + def raise_error(*args, **kwargs): + raise taskqueue.BadTaskStateError() + + queue_add_mock.side_effect = raise_error + tasks = (taskqueue.Task('A'), taskqueue.Task('1'), taskqueue.Task('B')) + + inserted = _insert_tasks(tasks, 'AbCd') + self.assertEqual(5, queue_add_mock.call_count) + self.assertEqual(0, inserted) @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) - def test_task_add_error(self, queue_add_mock): - """Ensure an exception doesn't get raised from add.""" + def test_task_add_error_TaskAlreadyExistsError(self, queue_add_mock): + """Ensure a TaskAlreadyExistsError doesn't get raised from add.""" from furious.context.context import _insert_tasks - def raise_transient(*args, **kwargs): + def raise_error(*args, **kwargs): from google.appengine.api import taskqueue - raise taskqueue.TransientError() + raise taskqueue.TaskAlreadyExistsError() - queue_add_mock.side_effect = raise_transient + queue_add_mock.side_effect = raise_error - _insert_tasks(('A',), 'AbCd') + inserted = _insert_tasks(('A',), 'AbCd') queue_add_mock.assert_called_once_with(('A',), transactional=False) + self.assertEqual(0, inserted) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_batches_get_split_TaskAlreadyExistsError(self, queue_add_mock): + """Ensure a batches get split and retried on TaskAlreadyExistsErrors. + """ + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + def raise_error(*args, **kwargs): + raise taskqueue.TaskAlreadyExistsError() + + queue_add_mock.side_effect = raise_error + tasks = (taskqueue.Task('A'), taskqueue.Task('1'), taskqueue.Task('B')) + + inserted = _insert_tasks(tasks, 'AbCd') + self.assertEqual(5, queue_add_mock.call_count) + self.assertEqual(0, inserted) @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) - def test_batches_get_split(self, queue_add_mock): - """Ensure a batches get split and retried on errors.""" + def test_task_add_error_TombstonedTaskError(self, queue_add_mock): + """Ensure a TombstonedTaskError doesn't get raised from add.""" from furious.context.context import _insert_tasks - def raise_transient(*args, **kwargs): + def raise_error(*args, **kwargs): from google.appengine.api import taskqueue - raise taskqueue.TransientError() + raise taskqueue.TombstonedTaskError() + + queue_add_mock.side_effect = raise_error + + inserted = _insert_tasks(('A',), 'AbCd') + queue_add_mock.assert_called_once_with(('A',), transactional=False) + self.assertEqual(0, inserted) - queue_add_mock.side_effect = raise_transient + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_batches_get_split_TombstonedTaskError(self, queue_add_mock): + """Ensure a batches get split and retried on TombstonedTaskErrors.""" + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + def raise_error(*args, **kwargs): + raise taskqueue.TombstonedTaskError() - _insert_tasks(('A', 1, 'B'), 'AbCd') + queue_add_mock.side_effect = raise_error + tasks = (taskqueue.Task('A'), taskqueue.Task('1'), taskqueue.Task('B')) + + inserted = _insert_tasks(tasks, 'AbCd') self.assertEqual(5, queue_add_mock.call_count) + self.assertEqual(0, inserted) + + @patch('time.sleep') + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_single_task_gets_retried(self, queue_add_mock, mock_sleep): + """Ensure a single task failing causes a retry. + """ + + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + queue_add_mock.side_effect = (taskqueue.TransientError, None) + + inserted = _insert_tasks((taskqueue.Task('A'),), 'AbCd') + self.assertEqual(queue_add_mock.call_count, 2) + self.assertEqual(inserted, 1) + + @patch('time.sleep') + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_tasks_not_enqueued_get_retried(self, queue_add_mock, mock_sleep): + """Ensure if a taskqueue add causes a TransientError, only the tasks + which were not enqueued are retried. + """ + + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + queue_add_mock.side_effect = (taskqueue.TransientError, None) + tasks = (Mock(), taskqueue.Task('1'), taskqueue.Task('B')) + tasks[0].was_enqueued = True + + inserted = _insert_tasks(tasks, 'AbCd') + + calls = [ + call(tasks, transactional=False), + call([tasks[1], tasks[2]], transactional=False) + ] + self.assertEqual(queue_add_mock.call_args_list, calls) + self.assertEqual(inserted, 3) + + @patch('time.sleep') + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_single_task_reraises_after_failure(self, queue_add_mock, + mock_sleep): + """Ensure a repeated failure on add re-raises the exception after + retrying. + """ + + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + queue_add_mock.side_effect = taskqueue.TransientError + + self.assertRaises(taskqueue.TransientError, + _insert_tasks, (taskqueue.Task('A'),), 'AbDc') + self.assertEqual(queue_add_mock.call_count, 2) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_batches_get_split_dont_reinsert_enqueued(self, queue_add_mock): + """Ensure a batch gets split and retried on error but only tasks which + were not enqueued are retried. + """ + from furious.context.context import _insert_tasks + from google.appengine.api import taskqueue + + def raise_error(*args, **kwargs): + raise taskqueue.TombstonedTaskError() + + queue_add_mock.side_effect = raise_error + tasks = (Mock(), taskqueue.Task('1'), taskqueue.Task('B')) + tasks[0].was_enqueued = True + + inserted = _insert_tasks(tasks, 'AbCd') + self.assertEqual(3, queue_add_mock.call_count) + self.assertEqual(1, inserted) + + +class TestTaskBatcher(unittest.TestCase): + + def test_no_tasks(self): + """Ensure that when to tasks are passed in, no tasks are returned.""" + from furious.context.context import _task_batcher + + self.assertEqual([], list(_task_batcher([]))) + + def test_one_task(self): + """Ensure that when one task is passed in, only one batch is returned + with one task in it. + """ + from furious.context.context import _task_batcher + + tasks = [1] + + result = list(_task_batcher(tasks)) + + self.assertEqual(1, len(result)) + self.assertEqual(1, len(result[0])) + + def test_less_than_100_tasks(self): + """Ensure that when less than 100 tasks are passed in, only one batch + is returned with all the tasks in it. + """ + from furious.context.context import _task_batcher + + tasks = 'a' * 99 + + result = list(_task_batcher(tasks)) + + self.assertEqual(1, len(result)) + self.assertEqual(len(tasks), len(result[0])) + + def test_more_than_100_tasks(self): + """Ensure that when more than 100 tasks are passed in, that the + correct number of batches are returned with the tasks in them. + """ + from furious.context.context import _task_batcher + + tasks = 'a' * 101 + + result = list(_task_batcher(tasks)) + + self.assertEqual(2, len(result)) + self.assertEqual(100, len(result[0])) + self.assertEqual(1, len(result[1])) + + def test_tasks_with_small_batch_size(self): + """Ensure that when a batch_size parameter is smaller than 100, + that the correct number of batches are created with the tasks in them. + """ + from furious.context.context import _task_batcher + + tasks = 'a' * 101 + batch_size = 30 + + result = list(_task_batcher(tasks, batch_size=batch_size)) + + self.assertEqual(4, len(result)) + self.assertEqual(30, len(result[0])) + self.assertEqual(30, len(result[1])) + self.assertEqual(30, len(result[2])) + self.assertEqual(11, len(result[3])) + + def test_more_than_100_tasks_with_large_batch_size(self): + """Ensure that when more than 100 tasks are passed in, and the + batch_size parameter is larger than 100, that batches with a max size + of 100 are returned with the tasks in them. + """ + from furious.context.context import _task_batcher + + tasks = 'a' * 101 + batch_size = 2000 + + result = list(_task_batcher(tasks, batch_size=batch_size)) + + self.assertEqual(2, len(result)) + self.assertEqual(100, len(result[0])) + self.assertEqual(1, len(result[1])) diff --git a/furious/tests/context/test_execution_context.py b/furious/tests/context/test_execution_context.py index b6febbe..0b50407 100644 --- a/furious/tests/context/test_execution_context.py +++ b/furious/tests/context/test_execution_context.py @@ -138,9 +138,9 @@ def test_job_removed_from_end_of_local_context(self): def test_corrupt_context(self): """Ensure wrong context is not popped from execution context stack.""" from furious.async import Async - from furious.context._execution import CorruptContextError from furious.context._execution import _ExecutionContext from furious.context._local import get_local_context + from furious.errors import CorruptContextError with self.assertRaises(CorruptContextError) as cm: job_outer = Async(target=dir) @@ -179,8 +179,8 @@ def test_async_type_required_as_first_argument(self): def test_double_init_raises_error(self): """Ensure initing twice raises a ContextExistsError.""" from furious.async import Async - from furious.context._execution import ContextExistsError from furious.context._execution import execution_context_from_async + from furious.errors import ContextExistsError execution_context_from_async(Async(target=dir)) self.assertRaises( diff --git a/furious/tests/context/test_local.py b/furious/tests/context/test_local.py new file mode 100644 index 0000000..045abba --- /dev/null +++ b/furious/tests/context/test_local.py @@ -0,0 +1,61 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest + + +class TestLocalContext(unittest.TestCase): + """Test that context._local functions correctly.""" + + def setUp(self): + """Setup the environment for a furious context.""" + + import os + import uuid + + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + def tearDown(self): + """Cleanup our environment modifications.""" + + import os + + del os.environ['REQUEST_ID_HASH'] + + def test_clear_context(self): + """Ensure clear_context successfully clears attributes set during + initialization from the local context. + """ + + from furious.context import _local + from furious.context._local import _clear_context + from furious.context._local import get_local_context + + # Initialize the local context by retrieving it. + get_local_context() + + # Make sure there is something on the local context we need to clear. + self.assertTrue(hasattr(_local._local_context, 'registry')) + + _clear_context() + + # Make sure local context entries have been cleared + self.assertFalse(hasattr(_local._local_context, '_executing_async')) + self.assertFalse( + hasattr(_local._local_context, '_executing_async_context')) + self.assertFalse(hasattr(_local._local_context, '_initialized')) + self.assertFalse(hasattr(_local._local_context, 'registry')) + diff --git a/furious/tests/dummy_module/__init__.py b/furious/tests/dummy_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/furious/tests/dummy_module/dumb.py b/furious/tests/dummy_module/dumb.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/furious/tests/dummy_module/dumb.py @@ -0,0 +1 @@ + diff --git a/furious/tests/extras/__init__.py b/furious/tests/extras/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/furious/tests/extras/appengine/__init__.py b/furious/tests/extras/appengine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/furious/tests/extras/appengine/test_ndb_persistence.py b/furious/tests/extras/appengine/test_ndb_persistence.py new file mode 100644 index 0000000..9c94078 --- /dev/null +++ b/furious/tests/extras/appengine/test_ndb_persistence.py @@ -0,0 +1,578 @@ +# +# Copyright 2014 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import os +import unittest + +from google.appengine.ext import testbed +from google.appengine.datastore import datastore_stub_util +from google.appengine.runtime.apiproxy_errors import DeadlineExceededError + +from mock import Mock +from mock import patch + +from furious.async import Async +from furious.async import AsyncResult + +from furious.context import Context + +from furious.processors import encode_exception + +from furious.extras.appengine.ndb_persistence import context_completion_checker +from furious.extras.appengine.ndb_persistence import ContextResult +from furious.extras.appengine.ndb_persistence import _completion_checker +from furious.extras.appengine.ndb_persistence import FuriousAsyncMarker +from furious.extras.appengine.ndb_persistence import FuriousContext +from furious.extras.appengine.ndb_persistence import FuriousCompletionMarker +from furious.extras.appengine.ndb_persistence import iter_context_results +from furious.extras.appengine.ndb_persistence import store_async_marker +from furious.extras.appengine.ndb_persistence import store_async_result +from furious.extras.appengine.ndb_persistence import store_context +from furious.extras.appengine.ndb_persistence import _check_markers + + +HRD_POLICY_PROBABILITY = 1 + + +class NdbTestBase(unittest.TestCase): + + def setUp(self): + super(NdbTestBase, self).setUp() + + os.environ['TZ'] = "UTC" + + self.testbed = testbed.Testbed() + self.testbed.activate() + self.testbed.setup_env(app_id="furious") + + self.policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( + probability=HRD_POLICY_PROBABILITY) + self.testbed.init_datastore_v3_stub(consistency_policy=self.policy) + self.testbed.init_memcache_stub() + + # TODO: Kill this + marker = FuriousAsyncMarker.query().fetch(1) + self.assertEqual(marker, []) + + def tearDown(self): + self.testbed.deactivate() + + super(NdbTestBase, self).tearDown() + + +class QueueTestCase(NdbTestBase): + + def test_queue_assignment(self): + from furious.extras.appengine.ndb_persistence import _get_current_queue + + queue = "test" + self.testbed.setup_env(HTTP_X_APPENGINE_QUEUENAME=queue) + result = _get_current_queue() + self.assertEqual(result, queue) + + +class ContextCompletionCheckerTestCase(NdbTestBase): + + def test_completion_store(self): + """Ensure the marker is stored on completion even without a result.""" + + async = Async('foo') + async._executed = True + + result = context_completion_checker(async) + + self.assertTrue(result) + + marker = FuriousAsyncMarker.get_by_id(async.id) + + self.assertIsNotNone(marker) + self.assertEqual(marker.key.id(), async.id) + self.assertEqual(marker.status, -1) + + def test_completion_store_with_result(self): + """Ensure the marker is stored on completion with a result.""" + + async = Async('foo') + async._executing = True + async.result = AsyncResult(status=1) + async._executed = True + + result = context_completion_checker(async) + + self.assertTrue(result) + + marker = FuriousAsyncMarker.get_by_id(async.id) + + self.assertIsNotNone(marker) + self.assertEqual(marker.key.id(), async.id) + self.assertEqual(marker.status, 1) + + +class StoreContextTestCase(NdbTestBase): + + def test_save_context(self): + """Ensure the passed in context gets serialized and set on the saved + FuriousContext entity. + """ + _id = "contextid" + + context = Context(id=_id) + + result = store_context(context) + + self.assertEqual(result.id(), _id) + + loaded_context = FuriousContext.from_id(result.id()) + + self.assertEqual(context.to_dict(), loaded_context.to_dict()) + + +class StoreAsyncMarkerTestCase(NdbTestBase): + + def test_marker_does_not_exist(self): + """Ensure the marker is saved if it does not already exist.""" + async_id = "asyncid" + + store_async_marker(async_id, 0) + + self.assertIsNotNone(FuriousAsyncMarker.get_by_id(async_id)) + + def test_marker_does_exist(self): + """Ensure the marker is not saved if it already exists.""" + async_id = "asyncid" + result = '{"foo": "bar"}' + FuriousAsyncMarker(id=async_id, result=result, status=1).put() + + store_async_marker(async_id, 1) + + marker = FuriousAsyncMarker.get_by_id(async_id) + + self.assertEqual(marker.result, result) + self.assertEqual(marker.status, 1) + + def test_store_async_exception(self): + """Ensure an async exception is encoded correctly.""" + async_id = "asyncid" + async_result = AsyncResult() + try: + raise Exception() + except Exception, e: + async_result.payload = encode_exception(e) + async_result.status = async_result.ERROR + + store_async_result(async_id, async_result) + + marker = FuriousAsyncMarker.get_by_id(async_id) + + self.assertEqual(marker.result, json.dumps(async_result.to_dict())) + self.assertEqual(marker.status, async_result.ERROR) + + +@patch('furious.extras.appengine.ndb_persistence._check_markers') +@patch.object(FuriousContext, 'from_id') +class CompletionCheckerTestCase(NdbTestBase): + + def test_markers_not_complete(self, context_from_id, check_markers): + """Ensure if not all markers are complete that False is returned and + the completion handler and cleanup tasks are not triggered. + """ + complete_event = Mock() + context = Context(id="contextid", + callbacks={'complete': complete_event}) + + context_from_id.return_value = context + + check_markers.return_value = False, False + + async = Async('foo') + async.update_options(context_id='contextid') + + result = _completion_checker(async.id, async.context_id) + + self.assertFalse(result) + + self.assertTrue(context_from_id.called) + + self.assertFalse(complete_event.start.called) + + def test_no_context_id(self, context_from_id, check_markers): + """Ensure if no context id that nothing happens. + """ + result = _completion_checker("1", None) + + self.assertIsNone(result) + + self.assertFalse(context_from_id.called) + + self.assertFalse(check_markers.start.called) + + def test_markers_complete(self, context_from_id, check_markers): + """Ensure if all markers are complete that True is returned and the + completion handler and cleanup tasks are triggered. + """ + complete_event = Mock() + context = Context(id="contextid", + callbacks={'complete': complete_event}) + + context_from_id.return_value = context + + check_markers.return_value = True, False + + async = Async('foo') + async.update_options(context_id='contextid') + FuriousCompletionMarker(id=async.context_id, complete=False).put() + + result = _completion_checker(async.id, async.context_id) + + self.assertTrue(result) + + complete_event.start.assert_called_once_with(transactional=True) + + @patch('furious.extras.appengine.ndb_persistence._mark_context_complete') + def test_markers_and_context_complete(self, mark, context_from_id, + check_markers): + """Ensure if all markers are complete that True is returned and + nothing else is done. + """ + async = Async('foo') + async.update_options(context_id='contextid') + + complete_event = Mock() + context = Context(id="contextid", + callbacks={'complete': complete_event}) + + context_from_id.return_value = context + + marker = FuriousCompletionMarker(id="contextid", complete=True) + marker.put() + + check_markers.return_value = True, False + mark.return_value = True + + result = _completion_checker(async.id, async.context_id) + + self.assertTrue(result) + + self.assertFalse(complete_event.start.called) + + marker.key.delete() + + @patch('furious.extras.appengine.ndb_persistence._insert_post_complete_tasks') + def test_marker_not_complete_when_start_fails(self, mock_insert, + context_from_id, + check_markers): + """Ensure if the completion handler fails to start, that the marker + does not get marked as complete. + """ + + complete_event = Mock() + context = Context(id="contextid", + callbacks={'complete': complete_event}) + + context_from_id.return_value = context + + check_markers.return_value = True, False + + async = Async('foo') + async.update_options(context_id='contextid') + FuriousCompletionMarker(id=async.context_id, complete=False).put() + + # Simulate the task failing to start + mock_insert.side_effect = DeadlineExceededError() + + self.assertRaises(DeadlineExceededError, + _completion_checker, async.id, async.context_id) + + # Marker should not have been marked complete. + current_marker = FuriousCompletionMarker.get_by_id(async.context_id) + self.assertFalse(current_marker.complete) + + +@patch('furious.extras.appengine.ndb_persistence.ndb.get_multi') +class CheckMarkersTestCase(NdbTestBase): + + def test_all_markers_exist(self, get_multi): + """Ensure True is returned when all markers exist.""" + task_ids = map(lambda x: "task" + str(x), range(11)) + + get_multi.side_effect = [Mock()], [Mock()] + + done, has_errors = _check_markers(task_ids) + + self.assertTrue(done) + self.assertFalse(has_errors) + + def test_not_all_markers_exist(self, get_multi): + """Ensure False is returned when not all markers exist.""" + task_ids = map(lambda x: "task" + str(x), range(11)) + + get_multi.side_effect = [Mock()], [None] + + done, has_errors = _check_markers(task_ids) + + self.assertFalse(done) + self.assertFalse(has_errors) + + +@patch('furious.extras.appengine.ndb_persistence.ndb.get_multi_async') +class IterResultsTestCase(NdbTestBase): + + def test_more_results_than_batch_size(self, get_multi_async): + """Ensure all the results are yielded out when more than the batch + size. + """ + marker1 = _build_marker(payload="1", status=1) + marker2 = _build_marker(payload="2", status=1) + marker3 = _build_marker(payload="3", status=1) + + future_set_1 = [_build_future(marker1), + _build_future(marker2)] + future_set_2 = [_build_future(marker3)] + + get_multi_async.side_effect = future_set_1, future_set_2 + + context = Context(_task_ids=["1", "2", "3"]) + + results = list(iter_context_results(context, batch_size=2)) + + self.assertEqual(results[0], ("1", marker1)) + self.assertEqual(results[1], ("2", marker2)) + self.assertEqual(results[2], ("3", marker3)) + + def test_less_results_than_batch_size(self, get_multi_async): + """Ensure all the results are yielded out when less than the batch + size. + """ + marker1 = _build_marker(payload="1", status=1) + marker2 = _build_marker(payload="2", status=1) + marker3 = _build_marker(payload="3", status=1) + + future_set_1 = [_build_future(marker1), _build_future(marker2), + _build_future(marker3)] + + get_multi_async.return_value = future_set_1 + + context = Context(_task_ids=["1", "2", "3"]) + + results = list(iter_context_results(context)) + + self.assertEqual(results[0], ("1", marker1)) + self.assertEqual(results[1], ("2", marker2)) + self.assertEqual(results[2], ("3", marker3)) + + def test_no_task_ids(self, get_multi_async): + """Ensure no results are yielded out when there are no task ids on the + passed in context. + """ + get_multi_async.return_value = [] + context = Context(_task_ids=[]) + + results = list(iter_context_results(context)) + + self.assertEqual(results, []) + + def test_keys_with_no_results(self, get_multi_async): + """Ensure empty results are yielded out when there are no items to + load but task ids are on the passed in context. + """ + future_set_1 = [_build_future(), _build_future(), _build_future()] + + get_multi_async.return_value = future_set_1 + + context = Context(_task_ids=["1", "2", "3"]) + + results = list(iter_context_results(context)) + + self.assertEqual(results[0], ("1", None)) + self.assertEqual(results[1], ("2", None)) + self.assertEqual(results[2], ("3", None)) + + def test_failure_in_marker(self, get_multi_async): + """Ensure all the results are yielded out when less than the batch + size and a failure is included in the results. + """ + async_id = "1" + async_result = AsyncResult() + try: + raise Exception() + except Exception, e: + async_result.payload = encode_exception(e) + async_result.status = async_result.ERROR + + json_dump = json.dumps(async_result.to_dict()) + marker1 = FuriousAsyncMarker(id=async_id, result=json_dump, + status=async_result.status) + + marker2 = FuriousAsyncMarker( + result=json.dumps(AsyncResult(payload="2", status=1).to_dict())) + marker3 = FuriousAsyncMarker( + result=json.dumps(AsyncResult(payload="3", status=1).to_dict())) + + future_set_1 = [_build_future(marker1), _build_future(marker2), + _build_future(marker3)] + + get_multi_async.return_value = future_set_1 + + context = Context(_task_ids=["1", "2", "3"]) + context_result = ContextResult(context) + + results = list(context_result.items()) + + self.assertEqual(results[0], ("1", json.loads(json_dump)["payload"])) + self.assertEqual(results[1], ("2", "2")) + self.assertEqual(results[2], ("3", "3")) + + +class ContextResultTestCase(NdbTestBase): + + @patch('furious.extras.appengine.ndb_persistence.ndb.get_multi_async') + def test_results_with_no_tasks_loaded(self, get_multi_async): + """Ensure results loads the tasks and yields them out when no tasks are + cached. + """ + marker1 = _build_marker(payload="1", status=1) + marker2 = _build_marker(payload="2", status=1) + marker3 = _build_marker(payload="3", status=1) + + future_set_1 = [_build_future(marker1), _build_future(marker2), + _build_future(marker3)] + + get_multi_async.return_value = future_set_1 + + context = Context(_task_ids=["1", "2", "3"]) + context_result = ContextResult(context) + + results = list(context_result.items()) + + results = sorted(results) + + self.assertEqual(results, [("1", "1"), ("2", "2"), ("3", "3")]) + + self.assertEqual(context_result._task_cache, { + "1": marker1, + "2": marker2, + "3": marker3 + }) + + @patch('furious.extras.appengine.ndb_persistence.ndb.get_multi_async') + def test_results_with_tasks_loaded(self, get_multi_async): + """Ensure results uses the cached tasks and yields them out when tasks + are cached. + """ + marker1 = _build_marker(payload="1", status=1) + marker2 = _build_marker(payload="2", status=1) + marker3 = _build_marker(payload="3", status=1) + + context = Context(_task_ids=["1", "2", "3"]) + context_result = ContextResult(context) + + context_result._task_cache = { + "1": marker1, + "2": marker2, + "3": marker3 + } + + results = list(context_result.items()) + + results = sorted(results) + + self.assertEqual(results, [("1", "1"), ("2", "2"), ("3", "3")]) + + self.assertFalse(get_multi_async.called) + + @patch('furious.extras.appengine.ndb_persistence.ndb.get_multi_async') + def test_results_with_tasks_loaded_missing_result(self, get_multi_async): + """Ensure results uses the cached tasks and yields them out when tasks + are cached and there's no results. + """ + marker1 = FuriousAsyncMarker() + + context = Context(_task_ids=["1", "2", "3"]) + context_result = ContextResult(context) + + context_result._task_cache = { + "1": marker1, + "2": None, + "3": None + } + + results = list(context_result.items()) + + results = sorted(results) + + self.assertEqual(results, [("1", None), ("2", None), ("3", None)]) + + self.assertFalse(get_multi_async.called) + + def test_has_errors_with_marker_not_cached(self): + """Ensure returns the value from the marker when not cached.""" + context_id = 1 + + FuriousCompletionMarker(id=context_id, has_errors=True).put() + + context = Context(id=context_id) + context_result = ContextResult(context) + + self.assertIsNone(context_result._marker) + self.assertTrue(context_result.has_errors()) + + context_result._marker.key.delete() + + def test_has_errors_with_marker_cached(self): + """Ensure returns the value from the marker when cached.""" + context_id = 1 + + marker = FuriousCompletionMarker(id=context_id, has_errors=True) + marker.put() + + context = Context(id=context_id) + context._marker = marker + context_result = ContextResult(context) + + self.assertIsNone(context_result._marker) + self.assertTrue(context_result.has_errors()) + + marker.key.delete() + + def test_has_no_marker(self): + """Ensure returns False when no marker found.""" + context_id = 1 + + context = Context(id=context_id) + context_result = ContextResult(context) + + self.assertIsNone(context_result._marker) + self.assertFalse(context_result.has_errors()) + + +def _build_marker(payload=None, status=None): + return FuriousAsyncMarker(result=json.dumps( + { + 'payload': payload, + 'status': status + } + )) + + +def _build_future(result=None): + future = Mock() + + if not result: + future.get_result.return_value = None + else: + future.get_result.return_value = result + + return future diff --git a/furious/tests/extras/test_insert_task_handlers.py b/furious/tests/extras/test_insert_task_handlers.py new file mode 100644 index 0000000..7a0fca6 --- /dev/null +++ b/furious/tests/extras/test_insert_task_handlers.py @@ -0,0 +1,96 @@ +import unittest + +from google.appengine.api import taskqueue +from google.appengine.ext import testbed +from mock import Mock +from mock import patch + +from furious.extras.insert_task_handlers import \ + insert_tasks_ignore_duplicate_names + + +class TestInsertTasksIgnoreDuplicateNames(unittest.TestCase): + """Test that _insert_tasks_ignore_duplicate_names behaves as expected.""" + def setUp(self): + harness = testbed.Testbed() + harness.activate() + harness.init_taskqueue_stub() + + @patch('furious.extras.insert_task_handlers._insert_tasks') + def test_insert_tasks_ignore_duplicate_names_relays_parms(self, + mock_tasks): + """Ensure args and kwargs passed, are passed onto _insert_tasks as + expected. + """ + + insert_tasks_ignore_duplicate_names(['1'], 'queue', 'arg1', foo='bar') + + mock_tasks.assert_called_once_with(['1'], 'queue', 'arg1', foo='bar') + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_insert_tasks_ignore_duplicate_names_raises(self, mock_add): + """Ensure if `DuplicateTaskNameError` raised, that each task is + retried. + + Also ensure that TaskAlreadyExistsError is not re-raised on the + subsequent retries. + """ + + mock_add.side_effect = (taskqueue.DuplicateTaskNameError, + taskqueue.TaskAlreadyExistsError, + taskqueue.TaskAlreadyExistsError, + taskqueue.TaskAlreadyExistsError) + + tasks = [taskqueue.Task('1'), taskqueue.Task('2'), taskqueue.Task('3')] + inserted = insert_tasks_ignore_duplicate_names(tasks, 'queue') + + # Ensure an attempt to re-insert all tasks. + self.assertEqual(inserted, 0) + self.assertEqual(mock_add.call_count, 4) + mock_add.assert_any_call(tasks, transactional=False) + for task in tasks: + mock_add.assert_any_call([task], transactional=False) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_insert_tasks_ignore_duplicate_names_partial_errors(self, mock_add): + """Ensure if `DuplicateTaskNameError` raised, that each task is + retried, and count is correct if some tasks could be re-inserted. + """ + + # Bulk add fails, then 2nd named task fails. + mock_add.side_effect = (taskqueue.DuplicateTaskNameError, + None, + taskqueue.TaskAlreadyExistsError, + None) + + tasks = [taskqueue.Task('1'), taskqueue.Task('2'), taskqueue.Task('3')] + inserted = insert_tasks_ignore_duplicate_names(tasks, 'queue') + + self.assertEqual(inserted, 2) + self.assertEqual(mock_add.call_count, 4) + mock_add.assert_any_call(tasks, transactional=False) + for task in tasks: + mock_add.assert_any_call([task], transactional=False) + + @patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_insert_tasks_ignore_duplicate_names_was_enqueued(self, mock_add): + """Ensure only tasks that were not enqueued on the initial BulkAdd() are + re-inserted. + """ + + # Bulk add fails, then 2nd named task fails. + mock_add.side_effect = (taskqueue.DuplicateTaskNameError, + taskqueue.TaskAlreadyExistsError, + None) + tasks = [Mock(), taskqueue.Task('2'), taskqueue.Task('3')] + tasks[0].was_enqueued = True + + inserted = insert_tasks_ignore_duplicate_names(tasks, 'queue') + + self.assertEqual(inserted, 2) + self.assertEqual(mock_add.call_count, 3) + + # No attempt to re-insert tasks[0], it was marked as 'enqueued'. + mock_add.assert_any_call(tasks, transactional=False) + mock_add.assert_any_call([tasks[1]], transactional=False) + mock_add.assert_any_call([tasks[2]], transactional=False) diff --git a/furious/tests/extras/test_xsrf.py b/furious/tests/extras/test_xsrf.py new file mode 100644 index 0000000..4556dfe --- /dev/null +++ b/furious/tests/extras/test_xsrf.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +import base64 + +from furious.extras import xsrf + +import unittest + + +class TestXSRFToken(unittest.TestCase): + def test_verify_timeout(self): + token = xsrf.XSRFToken('user@example.com', + 'secret', + current_time=1354160000) + token_string = token.generate_token_string() + token.verify_token_string(token_string, + timeout=10, + current_time=1354160010) + self.assertRaises(xsrf.XSRFTokenExpiredException, + token.verify_token_string, + token_string, + timeout=10, + current_time=1354160011) + + def test_verify_no_action(self): + token = xsrf.XSRFToken('user@example.com', + 'secret', + current_time=1354160000) + token_string = token.generate_token_string() + token.verify_token_string(token_string) + self.assertRaises( + xsrf.XSRFTokenInvalid, + token.verify_token_string, + xsrf.XSRFToken('user@example.com', + 'differentsecret', + current_time=1354160000).generate_token_string()) + self.assertRaises( + xsrf.XSRFTokenInvalid, + token.verify_token_string, + xsrf.XSRFToken('user@example.com', + 'secret', + current_time=1354160000).generate_token_string( + 'action')) + + def test_verify_action(self): + token = xsrf.XSRFToken('user@example.com', + 'secret', + current_time=1354160000) + token_string = token.generate_token_string('action') + token.verify_token_string(token_string, 'action') + self.assertRaises( + xsrf.XSRFTokenInvalid, + token.verify_token_string, + xsrf.XSRFToken('user@example.com', + 'differentsecret', + current_time=1354160000).generate_token_string()) + + def test_verify_substring(self): + """Tests that a substring of the correct token fails to verify.""" + token = xsrf.XSRFToken('user@example.com', + 'secret', + current_time=1354160000) + token_string = token.generate_token_string() + test_token, test_time = base64.urlsafe_b64decode(token_string).split('|') + test_string = base64.urlsafe_b64encode('|'.join([test_token[:-1], + test_time])) + self.assertRaises(xsrf.XSRFTokenInvalid, + token.verify_token_string, + test_string) + + def test_verify_bad_base_64(self): + token = xsrf.XSRFToken('user@example.com', + 'secret') + self.assertRaises( + xsrf.XSRFTokenMalformed, + token.verify_token_string, + 'wrong!!') + + def test_verify_no_delimiter(self): + token = xsrf.XSRFToken('user@example.com', + 'secret') + self.assertRaises( + xsrf.XSRFTokenMalformed, + token.verify_token_string, + base64.b64encode('NODELIMITER')) + + def test_verify_time_not_int(self): + token = xsrf.XSRFToken('user@example.com', + 'secret') + self.assertRaises( + xsrf.XSRFTokenMalformed, + token.verify_token_string, + base64.b64encode('NODE|NOTINT')) + diff --git a/furious/tests/handlers/__init__.py b/furious/tests/handlers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/furious/tests/handlers/test__init__.py b/furious/tests/handlers/test__init__.py new file mode 100644 index 0000000..b2735f3 --- /dev/null +++ b/furious/tests/handlers/test__init__.py @@ -0,0 +1,77 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import unittest + +from mock import patch + + +@patch('time.time') +@patch('logging.debug') +class TestLogTaskInfo(unittest.TestCase): + """Ensure that _log_task_info works as expected.""" + + def test_all_headers(self, debug_mock, time_mock): + """Ensure _log_task_info produces the correct logs.""" + from furious import handlers + time_mock.return_value = 1.5 + headers = { + 'X-Appengine-Taskretrycount': 'blue', + 'X-Appengine-Taskexecutioncount': 'yellow', + 'X-Appengine-Tasketa': '0.50' + } + + handlers._log_task_info(headers) + + expected_logs = ( + '{"ran": 1.5, "retry_count": "blue", "gae_latency_seconds": 1.0, ' + '"task_eta": 0.5, "execution_count": "yellow"}') + + debug_mock.assert_called_with('TASK-INFO: %s', expected_logs) + + def test_no_headers(self, debug_mock, time_mock): + """Ensure _log_task_info produces the correct logs.""" + from furious import handlers + time_mock.return_value = 1.5 + headers = {} + + handlers._log_task_info(headers) + + expected_logs = ( + '{"ran": 1.5, "retry_count": "", "gae_latency_seconds": 1.5, ' + '"task_eta": 0.0, "execution_count": ""}') + + debug_mock.assert_called_with('TASK-INFO: %s', expected_logs) + + def test_ignore_extra_headers(self, debug_mock, time_mock): + """Ensure _log_task_info ignores extra items in the header""" + from furious import handlers + time_mock.return_value = 1.5 + headers = { + 'something-something': 'please-ignore-me', + 'X-Appengine-Taskretrycount': 'blue', + 'X-Appengine-Taskexecutioncount': 'yellow', + 'X-Appengine-Tasketa': '0.50' + } + + handlers._log_task_info(headers) + + expected_logs = ( + '{"ran": 1.5, "retry_count": "blue", "gae_latency_seconds": 1.0, ' + '"task_eta": 0.5, "execution_count": "yellow"}') + + debug_mock.assert_called_with('TASK-INFO: %s', expected_logs) diff --git a/furious/tests/test_async.py b/furious/tests/test_async.py index ac71217..9241757 100644 --- a/furious/tests/test_async.py +++ b/furious/tests/test_async.py @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from collections import deque +import functools +import unittest import json +import mock -import unittest - -from mock import patch +from furious.async import Async class TestDefaultsDecorator(unittest.TestCase): @@ -93,19 +95,34 @@ def some_method(): class TestAsync(unittest.TestCase): """Make sure Async produces correct Task objects.""" + def setUp(self): + import os + import uuid + from furious.context import _local + + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + local_context = _local.get_local_context() + local_context._executing_async_context = None + + def tearDown(self): + import os + + del os.environ['REQUEST_ID_HASH'] + def test_none_function(self): """Ensure passing None as function raises.""" from furious.async import Async - from furious.job_utils import BadFunctionPathError + from furious.errors import BadObjectPathError - self.assertRaises(BadFunctionPathError, Async, None) + self.assertRaises(BadObjectPathError, Async, None) def test_empty_function_path(self): """Ensure passing None as function raises.""" from furious.async import Async - from furious.job_utils import BadFunctionPathError + from furious.errors import BadObjectPathError - self.assertRaises(BadFunctionPathError, Async, '') + self.assertRaises(BadObjectPathError, Async, '') def test_job_params(self): """Ensure good args and kwargs generate a well-formed job tuple.""" @@ -114,7 +131,7 @@ def test_job_params(self): job = ("test", [1, 2, 3], {'a': 1, 'b': 2, 'c': 3}) async_job = Async(*job) - self.assertEqual(job, async_job._options['job']) + self.assertEqual(job, async_job.job) def test_no_args_or_kwargs(self): """Ensure no args and no kwargs generate a well-formed job tuple.""" @@ -124,7 +141,7 @@ def test_no_args_or_kwargs(self): async_job = Async(function) self.assertEqual(function, async_job._function_path) - self.assertEqual((function, None, None), async_job._options['job']) + self.assertEqual((function, None, None), async_job.job) def test_args_with_no_kwargs(self): """Ensure args and no kwargs generate a well-formed job tuple.""" @@ -133,7 +150,7 @@ def test_args_with_no_kwargs(self): job = ("test", (1, 2, 3)) async_job = Async(*job) - self.assertEqual(job + (None,), async_job._options['job']) + self.assertEqual(job + (None,), async_job.job) def test_no_args_with_kwargs(self): """Ensure no args with kwargs generate a well-formed job tuple.""" @@ -142,7 +159,7 @@ def test_no_args_with_kwargs(self): job = ("test", None, {'a': 1, 'b': 'c', 'alpha': True}) async_job = Async(*job) - self.assertEqual(job, async_job._options['job']) + self.assertEqual(job, async_job.job) def test_gets_callable_path(self): """Ensure the job tuple contains the callable path.""" @@ -157,7 +174,7 @@ def some_function(): self.assertEqual( ('furious.tests.test_async.some_function',) + job_args, - async_job._options['job']) + async_job.job) def test_none_args_and_kwargs(self): """Ensure args and kwargs may be None.""" @@ -166,14 +183,73 @@ def test_none_args_and_kwargs(self): job = ("something", None, None,) async_job = Async(*job) - self.assertEqual(job, async_job._options['job']) + self.assertEqual(job, async_job.job) + + @mock.patch('uuid.uuid4', autospec=True) + def test_generates_id(self, uuid_patch): + """Ensure an id is auto-generated if not specified.""" + from furious.async import Async + + id = 'random-id' + uuid_patch.return_value.hex = id + + job = Async('somehting') + + self.assertEqual(job.id, id) + self.assertEqual(job.get_options()['id'], id) + + def test_generates_one_id(self): + """Ensure only one random id is auto-generated if not specified.""" + from furious.async import Async + + job = Async('somehting') + + id1 = job.id + id2 = job.id + self.assertEqual(id1, id2) + self.assertEqual(job.id, id1) + + def test_uses_given_id(self): + """Ensure an id passed in is used.""" + from furious.async import Async + + job = Async('somehting', id='superrandom') + + self.assertEqual(job.id, 'superrandom') + self.assertEqual(job.get_options()['id'], 'superrandom') + + def test_update_id(self): + """Ensure using update options to update an id works.""" + from furious.async import Async + + job = Async('somehting') + job.update_options(id='newid') + + self.assertEqual(job.id, 'newid') + self.assertEqual(job.get_options()['id'], 'newid') + + def test_context_id(self): + """Ensure context_id returns the context_id.""" + from furious.async import Async + + job = Async('somehting') + job.update_options(context_id='blarghahahaha') + self.assertEqual(job.context_id, 'blarghahahaha') + + def test_no_context_id(self): + """Ensure calling context_id when none exists returns None.""" + from furious.async import Async + + job = Async('somehting') + self.assertIsNone(job.context_id) def test_decorated_options(self): """Ensure the defaults decorator sets Async options.""" from furious.async import Async from furious.async import defaults - options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} + options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}, 'id': 'thing', + 'context_id': None, 'parent_id': 'parentid'} @defaults(**options.copy()) def some_function(): @@ -182,6 +258,7 @@ def some_function(): job = Async(some_function) options['job'] = ("furious.tests.test_async.some_function", None, None) + options['_recursion'] = {'current': 0, 'max': 100} self.assertEqual(options, job._options) @@ -190,18 +267,21 @@ def test_init_opts_supersede_decorated_options(self): from furious.async import Async from furious.async import defaults - options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} + options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}, 'id': 'wrong', + 'context_id': None, 'parent_id': 'parentid'} @defaults(**options.copy()) def some_function(): pass - job = Async(some_function, value=17, other='abc') + job = Async(some_function, value=17, other='abc', id='correct') options['value'] = 17 options['other'] = 'abc' + options['id'] = 'correct' options['job'] = ("furious.tests.test_async.some_function", None, None) + options['_recursion'] = {'current': 0, 'max': 100} self.assertEqual(options, job._options) @@ -221,7 +301,7 @@ def test_set_execution_context_disallows_double_set(self): AlreadyInContextError. """ from furious.async import Async - from furious.context import AlreadyInContextError + from furious.errors import AlreadyInContextError async = Async(target=dir) async.set_execution_context(object()) @@ -232,30 +312,37 @@ def test_update_options(self): """Ensure update_options updates the options.""" from furious.async import Async - options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} + options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}, 'id': 'xx', + 'context_id': None, 'parent_id': 'parentid'} job = Async("nonexistant") job.update_options(**options.copy()) options['job'] = ("nonexistant", None, None) + options['_recursion'] = {'current': 0, 'max': 100} + self.assertEqual(options, job._options) def test_update_options_supersede_init_opts(self): """Ensure update_options supersedes the options set in init.""" from furious.async import Async - options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} + options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}, 'id': 'wrong', + 'context_id': None, 'parent_id': 'parentid'} job = Async("nonexistant", **options.copy()) - job.update_options(value=23, other='stuff') + job.update_options(value=23, other='stuff', id='right') options['value'] = 23 options['other'] = 'stuff' + options['id'] = 'right' options['job'] = ("nonexistant", None, None) + options['_recursion'] = {'current': 0, 'max': 100} + self.assertEqual(options, job._options) def test_get_options(self): @@ -325,17 +412,29 @@ def test_get_empty_task_args(self): self.assertEqual({}, job.get_task_args()) + def test_deepcopy(self): + """Make sure you can deepcopy an Async.""" + import copy + + from furious.async import Async + + job = Async(dir) + copy.deepcopy(job) + def test_to_dict(self): """Ensure to_dict returns a dictionary representation of the Async.""" from furious.async import Async task_args = {'other': 'zzz', 'nested': 1} headers = {'some': 'thing', 'fun': 1} - options = {'headers': headers, 'task_args': task_args} + options = {'headers': headers, 'task_args': task_args, 'id': 'me', + 'context_id': None, 'parent_id': 'parentid'} job = Async('nonexistant', **options.copy()) options['job'] = ('nonexistant', None, None) + options['_recursion'] = {'current': 0, 'max': 100} + options['_type'] = 'furious.async.Async' self.assertEqual(options, job.to_dict()) @@ -343,14 +442,15 @@ def test_to_dict_with_callbacks(self): """Ensure to_dict correctly encodes callbacks.""" from furious.async import Async - def success_function(): - pass - - options = {'callbacks': { - 'success': self.__class__.test_to_dict_with_callbacks, - 'failure': "failure_function", - 'exec': Async(target=dir) - }} + options = {'id': 'anident', + 'context_id': 'contextid', + 'parent_id': 'parentid', + 'callbacks': { + 'success': self.__class__.test_to_dict_with_callbacks, + 'failure': "failure_function", + 'exec': Async(target=dir, id='subidnet', + parent_id='parentid'), + }} job = Async('nonexistant', **options.copy()) @@ -359,8 +459,15 @@ def success_function(): 'success': ("furious.tests.test_async." "TestAsync.test_to_dict_with_callbacks"), 'failure': "failure_function", - 'exec': {'job': ('dir', None, None)} + 'exec': {'job': ('dir', None, None), + 'id': 'subidnet', + 'context_id': None, + 'parent_id': 'parentid', + '_recursion': {'current': 0, 'max': 100}, + '_type': 'furious.async.Async'} } + options['_recursion'] = {'current': 0, 'max': 100} + options['_type'] = 'furious.async.Async' self.assertEqual(options, job.to_dict()) @@ -379,6 +486,7 @@ def test_from_dict(self): self.assertEqual(headers, async_job.get_headers()) self.assertEqual(task_args, async_job.get_task_args()) self.assertEqual(job[0], async_job._function_path) + self.assertEqual(job[0], async_job.function_path) def test_from_dict_with_callbacks(self): """Ensure from_dict reconstructs callbacks correctly.""" @@ -389,10 +497,11 @@ def test_from_dict_with_callbacks(self): 'success': ("furious.tests.test_async." "TestAsync.test_to_dict_with_callbacks"), 'failure': "dir", - 'exec': {'job': ('dir', None, None)} + 'exec': {'job': ('dir', None, None), 'id': 'petey', + 'parent_id': 'parentid'} } - options = {'job': job, 'callbacks': callbacks} + options = {'job': job, 'callbacks': callbacks, 'parent_id': 'parentid'} async_job = Async.from_dict(options) @@ -404,8 +513,15 @@ def test_from_dict_with_callbacks(self): callbacks = async_job.get_callbacks() exec_callback = callbacks.pop('exec') + correct_options = {'job': ('dir', None, None), + 'id': 'petey', + 'parent_id': 'parentid', + 'context_id': None, + '_recursion': {'current': 0, 'max': 100}, + '_type': 'furious.async.Async'} + self.assertEqual(check_callbacks, callbacks) - self.assertEqual({'job': ('dir', None, None)}, exec_callback.to_dict()) + self.assertEqual(correct_options, exec_callback.to_dict()) def test_reconstitution(self): """Ensure to_dict(job.from_dict()) returns the same thing.""" @@ -414,7 +530,17 @@ def test_reconstitution(self): headers = {'some': 'thing', 'fun': 1} job = ('test', None, None) task_args = {'other': 'zzz', 'nested': 1} - options = {'job': job, 'headers': headers, 'task_args': task_args} + options = { + 'job': job, + 'id': 'someid', + 'headers': headers, + 'task_args': task_args, + 'persistence_engine': 'furious.extras.appengine.ndb_persistence', + '_recursion': {'current': 1, 'max': 100}, + '_type': 'furious.async.Async', + 'context_id': None, + 'parent_id': 'parentid' + } async_job = Async.from_dict(options) @@ -445,7 +571,9 @@ def test_to_task(self): expected_url = "%s/%s" % (ASYNC_ENDPOINT, 'test') task_args = {'eta': eta_posix} - options = {'job': job, 'headers': headers, 'task_args': task_args} + options = {'job': job, 'headers': headers, 'task_args': task_args, + 'id': 'ident', 'context_id': 'contextid', + 'parent_id': 'parentid'} task = Async.from_dict(options).to_task() @@ -460,13 +588,19 @@ def test_to_task(self): self.assertEqual(expected_url, task.url) self.assertEqual(full_headers, task.headers) + options['task_args']['eta'] = datetime.datetime.fromtimestamp( + eta_posix) + + options['_recursion'] = {'current': 1, 'max': 100} + options['_type'] = 'furious.async.Async' + self.assertEqual( options, Async.from_dict(json.loads(task.payload)).get_options()) def test_getting_result_fails(self): """Ensure attempting to get the result before executing raises.""" from furious.async import Async - from furious.async import NotExecutedError + from furious.errors import NotExecutedError job = Async(target=dir) @@ -490,7 +624,7 @@ def test_getting_result(self): def test_setting_result_fails(self): """Ensure the result can not be set without the execute flag set.""" from furious.async import Async - from furious.async import NotExecutingError + from furious.errors import NotExecutingError job = Async(target=dir) @@ -510,15 +644,541 @@ def test_setting_result(self): self.assertEqual(123, job.result) self.assertTrue(job.executed) - @patch('google.appengine.api.taskqueue.Queue', autospec=True) - def test_start(self, queue_mock): + def test_setting_result_does_not_call_persist(self): + """Ensure setting the result doesn't call persist result if not in + persist mode. + """ + from furious.async import Async + + result = "here be the results." + + persistence_engine = mock.Mock() + + job = Async(target=dir) + + # Manually set the persistence_engine so that the Async doesn't try to + # reload the mock persistence_engine. + job._persistence_engine = persistence_engine + + job.executing = True + job.result = result + + self.assertEqual(persistence_engine.store_async_result.call_count, 0) + + def test_setting_result_calls_persist(self): + """Ensure setting the result calls the persist_result method.""" + from furious.async import Async + + result = "here be the results." + + persistence_engine = mock.Mock() + + job = Async(target=dir, persist_result=True) + + # Manually set the persistence_engine so that the Async doesn't try to + # reload the mock persistence_engine. + job._persistence_engine = persistence_engine + + job.executing = True + job.result = result + + persistence_engine.store_async_result.assert_called_once_with(job.id, + result) + + @mock.patch('time.sleep') + @mock.patch('google.appengine.api.taskqueue.Queue', autospec=True) + def test_start_hits_transient_error(self, queue_mock, mock_sleep): + """Ensure the task retries if a transient error is hit.""" + from google.appengine.api.taskqueue import TransientError + from furious.async import Async + + def add(task, *args, **kwargs): + def add_second(task, *args, **kwargs): + assert task + + queue_mock.return_value.add.side_effect = add_second + raise TransientError() + + queue_mock.return_value.add.side_effect = add + + async_job = Async("something", queue='my_queue') + async_job.start() + + queue_mock.assert_called_with(name='my_queue') + self.assertEqual(2, queue_mock.return_value.add.call_count) + self.assertEqual(1, mock_sleep.call_count) + + @mock.patch('time.sleep') + @mock.patch('google.appengine.api.taskqueue.Queue', autospec=True) + def test_start_hits_transient_error_retry_disabled(self, queue_mock, + sleep_mock): + """Ensure if transient error retries are disabled, that those errors are + re-raised immediately without any attempt to re-insert. + """ + from google.appengine.api.taskqueue import TransientError + from furious.async import Async + + queue_mock.return_value.add.side_effect = TransientError() + + async_job = Async("something", queue='my_queue', + retry_transient_errors=False) + + self.assertRaises(TransientError, async_job.start) + self.assertEqual(1, queue_mock.return_value.add.call_count) + + # Try again with the option enabled, this should cause a retry after a + # delay, which we have also specified. + queue_mock.reset_mock() + async_job = Async("something", queue='my_queue', + retry_transient_errors=True, + retry_delay=12) + + self.assertRaises(TransientError, async_job.start) + self.assertEqual(2, queue_mock.return_value.add.call_count) + sleep_mock.assert_called_once_with(12) + + @mock.patch('time.sleep') + @mock.patch('google.appengine.api.taskqueue.Queue', autospec=True) + def test_start_hits_transient_error_transactional(self, queue_mock, + sleep_mock): + """Ensure if caller is specifying transactional, that Transient errors + are immediately re-raised. + """ + from google.appengine.api.taskqueue import TransientError + from furious.async import Async + + queue_mock.return_value.add.side_effect = TransientError() + + async_job = Async("something", queue='my_queue', + retry_transient_errors=True) + + self.assertRaises(TransientError, async_job.start, + transactional=True) + self.assertEqual(1, queue_mock.return_value.add.call_count) + self.assertEqual(0, sleep_mock.call_count) + + @mock.patch('google.appengine.api.taskqueue.Queue', autospec=True) + def test_start_hits_other_error_retry_enabled(self, queue_mock): + """Ensure if transient error retries are enabled, that other errors are + not retried. + """ + + from furious.async import Async + + queue_mock.return_value.add.side_effect = (Exception(), None) + + async_job = Async("something", queue='my_queue', + retry_transient_errors=True) + + self.assertRaises(Exception, async_job.start) + self.assertEqual(1, queue_mock.return_value.add.call_count) + + @mock.patch('google.appengine.api.taskqueue.Queue', autospec=True) + def test_start_hits_task_already_exists_error_error(self, queue_mock): + """Ensure the task returns if a task already exists error is hit.""" + from google.appengine.api.taskqueue import TaskAlreadyExistsError + from furious.async import Async + + queue_mock.return_value.add.side_effect = TaskAlreadyExistsError() + + async_job = Async("something", queue='my_queue') + async_job.start() + + queue_mock.assert_called_with(name='my_queue') + self.assertEqual(1, queue_mock.return_value.add.call_count) + + @mock.patch('google.appengine.api.taskqueue.Queue', autospec=True) + def test_start_hits_tombstoned_task_error_error(self, queue_mock): + """Ensure the task returns if a tombstoned task error is hit.""" + from google.appengine.api.taskqueue import TombstonedTaskError + from furious.async import Async + + queue_mock.return_value.add.side_effect = TombstonedTaskError() + + async_job = Async("something", queue='my_queue') + async_job.start() + + queue_mock.assert_called_with(name='my_queue') + self.assertEqual(1, queue_mock.return_value.add.call_count) + + @mock.patch('google.appengine.api.taskqueue.Queue', autospec=True) + def test_start_runs_successfully(self, queue_mock): """Ensure the Task is inserted into the specified queue.""" from furious.async import Async async_job = Async("something", queue='my_queue') - # task = async_job.to_task() async_job.start() + queue_mock.assert_called_once_with(name='my_queue') + self.assertTrue(queue_mock.return_value.add.called) + # TODO: Check that the task is the same. # self.assertEqual(task, queue_mock.add.call_args) + @mock.patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_task_transactional(self, queue_add_mock): + """Ensure the task is added transactional when start is + called with transactional.""" + from furious.async import Async + + async_job = Async("something") + async_job.start(transactional=True) + call_args = queue_add_mock.call_args + call_kwargs = call_args[1] + + self.assertIn('transactional', call_kwargs) + self.assertTrue(call_kwargs['transactional']) + + @mock.patch('google.appengine.api.taskqueue.Queue.add', auto_spec=True) + def test_task_non_transactional(self, queue_add_mock): + """Ensure the task is added transactional when start is + called with transactional.""" + from furious.async import Async + + async_job = Async("something") + async_job.start(transactional=False) + call_args = queue_add_mock.call_args + call_kwargs = call_args[1] + + self.assertIn('transactional', call_kwargs) + self.assertFalse(call_kwargs['transactional']) + + @mock.patch('google.appengine.api.taskqueue.Queue.add_async', + auto_spec=True) + def test_start_async_no_rpc(self, queue_add_async_mock): + """Ensure that when the task is called with async=True, that the + add_async method is called with the default rpc=None. + """ + from furious.async import Async + + async_job = Async("something") + async_job.start(async=True) + + self.assertTrue(queue_add_async_mock.called) + self.assertEqual(None, queue_add_async_mock.call_args[1]['rpc']) + + @mock.patch('google.appengine.api.taskqueue.Queue.add_async', + auto_spec=True) + def test_start_async_with_rpc(self, queue_add_async_mock): + """Ensure that when the task is called with async=True and an rpc is + provided, that the add_async method is called with the correct rpc. + """ + from mock import Mock + from furious.async import Async + + rpc = Mock() + async_job = Async("something") + async_job.start(async=True, rpc=rpc) + + self.assertTrue(queue_add_async_mock.called) + self.assertEqual(rpc, queue_add_async_mock.call_args[1]['rpc']) + + def test_update_recursion_level_defaults(self): + """Ensure that defaults (1, MAX_DEPTH) are set correctly.""" + from furious.async import Async + from furious.async import MAX_DEPTH + + async_job = Async("something") + + async_job._increment_recursion_level() + + options = async_job.get_options()['_recursion'] + self.assertEqual(1, options['current']) + self.assertEqual(MAX_DEPTH, options['max']) + + def test_check_recursion_level_execution_context(self): + """Ensure that when there is an existing Async that the correct values + are pulled and incremented from there, not the defaults. + """ + from furious.async import Async + from furious.context import execution_context_from_async + + context_async = Async("something", _recursion={'current': 42, + 'max': 77}) + new_async = Async("something_else") + + with execution_context_from_async(context_async): + new_async._increment_recursion_level() + + self.assertEqual(43, new_async.recursion_depth) + + options = new_async.get_options()['_recursion'] + self.assertEqual(77, options['max']) + + def test_check_recursion_level_overridden_interior_max(self): + """Ensure that when there is an existing Async that the correct values + are pulled and incremented from there, unless the interior Async sets + it's own custom max. + """ + from furious.async import Async + from furious.context import execution_context_from_async + + context_async = Async("something", _recursion={'current': 42, + 'max': 77}) + + new_async = Async("something_else", _recursion={'max': 89}) + + with execution_context_from_async(context_async): + new_async._increment_recursion_level() + + options = new_async.get_options()['_recursion'] + self.assertEqual(43, options['current']) + self.assertEqual(89, options['max']) + + def test_check_recursion_depth_over_limit(self): + """Ensure that when over the recusion limit, calling + check_recursion_depth raises a AsyncRecursionError. + """ + from furious.async import Async + from furious.errors import AsyncRecursionError + + async = Async("something", _recursion={'current': 8, 'max': 7}) + + self.assertRaises(AsyncRecursionError, async.check_recursion_depth) + + def test_check_recursion_disabled(self): + """Ensure that when recursion max depth is explicitly set to -1, then + the recursion check is disabled. + + There are no explicit asserts in this test because the + check_recursion_depth() method would throw an exception if this + functionality wasn't working. + """ + from furious.async import Async + + async_job = Async("something", _recursion={'current': 101, + 'max': -1}) + + async_job.check_recursion_depth() + + def test_retry_default(self): + """Ensure that when no task_retry_limit specified, that the default is + set. + """ + from furious.async import Async + from furious.async import MAX_RESTARTS + + async_job = Async("something") + task = async_job.to_task() + + self.assertEqual(MAX_RESTARTS, task.retry_options.task_retry_limit) + + def test_retry_custom(self): + """Ensure that when a custom retry limit is set, that it's + propagated. + """ + from furious.async import Async + + async_job = Async("something", + task_args={'retry_options': {'task_retry_limit': 5}}) + task = async_job.to_task() + + self.assertEqual(5, task.retry_options.task_retry_limit) + + def test_retry_value_without_to_task(self): + """Ensure that when you encode the options, the retry_options are not + affected. + """ + from furious.async import Async + from furious.async import encode_async_options + + async_job = Async("something", + task_args={'retry_options': {'task_retry_limit': 5}}) + options = encode_async_options(async_job) + + self.assertEqual( + 5, options['task_args']['retry_options']['task_retry_limit']) + + def test_retry_value_with_to_task(self): + """Ensure that calling to_task doesn't affect the options when + encoding. + """ + from furious.async import Async + from furious.async import encode_async_options + + async_job = Async("something", + task_args={'retry_options': {'task_retry_limit': 5}}) + async_job.to_task() + options = encode_async_options(async_job) + + self.assertEqual( + 5, options['task_args']['retry_options']['task_retry_limit']) + + def test_context_checker_encoded(self): + """Ensure the _context_checker is correctly encoded in options dict.""" + from furious.async import Async + from furious.async import encode_async_options + + async_job = Async("something", _context_checker=dir) + options = encode_async_options(async_job) + + self.assertEqual('dir', options['__context_checker']) + + def test_context_checker_encoded_and_decoded(self): + """Ensure the _context_checker is correctly encoded to and decoded from + an Async options dict. + """ + from furious.async import Async + + async_job = Async("something", _context_checker=dir) + + encoded_async = async_job.to_dict() + self.assertEqual(encoded_async['__context_checker'], 'dir') + + new_async_job = Async.from_dict(encoded_async) + self.assertEqual(new_async_job.get_options()['_context_checker'], dir) + + self.assertEqual(async_job.to_dict(), new_async_job.to_dict()) + + def test_process_results_encoded_and_decoded(self): + """Ensure _process_results is correctly encoded to and decoded from + an Async options dict. + """ + from furious.async import Async + + async_job = Async("something", _process_results=locals) + + encoded_async = async_job.to_dict() + self.assertEqual(encoded_async['__process_results'], 'locals') + + new_async_job = Async.from_dict(encoded_async) + self.assertEqual(new_async_job.get_options()['_process_results'], locals) + + self.assertEqual(async_job.to_dict(), new_async_job.to_dict()) + + def test_retry_value_is_decodable(self): + """Ensure that from_dict is the inverse of to_dict when retry options + are given. + """ + from furious.async import Async + + async_job = Async("something", + task_args={'retry_options': {'task_retry_limit': 5}}) + new_async_job = Async.from_dict(async_job.to_dict()) + + self.assertEqual(async_job.to_dict(), new_async_job.to_dict()) + + def test_used_async_retry_value_is_decodable(self): + """Ensure that from_dict is the inverse of to_dict when retry options + are given and the async has be cast to task. + """ + from furious.async import Async + + async_job = Async("something", + task_args={'retry_options': {'task_retry_limit': 5}}) + async_job.to_dict() + + new_async_job = Async.from_dict(async_job.to_dict()) + + self.assertEqual(async_job.to_dict(), new_async_job.to_dict()) + + +def _foo_stub(): + TestAsync_decorate_job.foo_values.append('bar') + + +class FooAsync(Async): + def _decorate_job(self): + return super(FooAsync, self)._decorate_job() + + +class FizzAsync(Async): + @staticmethod + def fizz(func): + @functools.wraps(func) + def wrapper(): + TestAsync_decorate_job.foo_values.append('fizz') + func() + TestAsync_decorate_job.foo_values.append('bang') + return wrapper + + def _decorate_job(self): + return self.fizz(super(FizzAsync, self)._decorate_job()) + + +class TestAsync_decorate_job(unittest.TestCase): + """Tests `Async._decorate_job`. + + These tests will run the tasks to ensure that the decorated function is + what gets invoked. + """ + foo_values = deque() + + def setUp(self): + from google.appengine.ext.testbed import ( + TASKQUEUE_SERVICE_NAME, + Testbed, + ) + testbed = Testbed() + testbed.activate() + + self.addCleanup(testbed.deactivate) + self.addCleanup(TestAsync_decorate_job.foo_values.clear) + + testbed.init_taskqueue_stub() + self._taskq_service = testbed.get_stub(TASKQUEUE_SERVICE_NAME) + + def run_taskq(self): + from furious.test_stubs.appengine.queues import run + run(self._taskq_service) + + def test_runs_undecorated_job(self): + """By default, doesn't decorate the function with anything.""" + FooAsync(_foo_stub).start() + self.run_taskq() + self.assertSequenceEqual(['bar'], TestAsync_decorate_job.foo_values) + + def test_runs_decorated_job(self): + """Subclass can override `Async._decorate_job` to wrap the original + target. + """ + FizzAsync(_foo_stub).start() + self.run_taskq() + self.assertSequenceEqual( + ['fizz', 'bar', 'bang'], TestAsync_decorate_job.foo_values) + + +class TestAsyncFromOptions(unittest.TestCase): + """Ensure async_from_options() works correctly.""" + + def setUp(self): + import os + import uuid + + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + def tearDown(self): + import os + + del os.environ['REQUEST_ID_HASH'] + + def test_no_type(self): + """Ensure that if not _type is in options, that it defaults to + furious.async.Async. + """ + from furious.async import Async + from furious.async import async_from_options + + async_job = Async(dir) + + options = async_job.to_dict() + options.pop('_type') + + result = async_from_options(options) + + self.assertIsInstance(result, Async) + + def test_has_type(self): + """Ensure that if _type is not furious.async.Async that the correct + subclass is instantiated. + """ + from furious.async import async_from_options + from furious.batcher import MessageProcessor + + async_job = MessageProcessor(dir) + + options = async_job.to_dict() + + result = async_from_options(options) + + self.assertIsInstance(result, MessageProcessor) diff --git a/furious/tests/test_batcher.py b/furious/tests/test_batcher.py index 1427933..5d415cb 100644 --- a/furious/tests/test_batcher.py +++ b/furious/tests/test_batcher.py @@ -1,37 +1,62 @@ import json import unittest -from mock import patch +from mock import Mock, patch class MessageTestCase(unittest.TestCase): - def test_options_are_set(self): + def test_id_is_passed_in_with_options(self): + """Ensure id is set with options passed to init are set on the message. + """ + from furious.batcher import Message + + _id = "id" + + options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}} + + message = Message(id=_id, value=1, other='zzz', nested={1: 1}) + + self.assertEqual(options, message._options) + + @patch('furious.batcher.uuid.uuid4') + def test_options_are_set(self, get_uuid): """Ensure options passed to init are set on the message.""" from furious.batcher import Message - options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} + _id = "id" + get_uuid.return_value.hex = _id + + options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}} message = Message(value=1, other='zzz', nested={1: 1}) self.assertEqual(options, message._options) - def test_update_options(self): + @patch('furious.batcher.uuid.uuid4') + def test_update_options(self, get_uuid): """Ensure update_options updates the options.""" from furious.batcher import Message - options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} + _id = "id" + get_uuid.return_value.hex = _id + + options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}} message = Message() message.update_options(**options.copy()) self.assertEqual(options, message._options) - def test_update_options_supersede_init_opts(self): + @patch('furious.batcher.uuid.uuid4') + def test_update_options_supersede_init_opts(self, get_uuid): """Ensure update_options supersedes the options set in init.""" from furious.batcher import Message - options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} + _id = "id" + get_uuid.return_value.hex = _id + + options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}} message = Message(**options) @@ -69,7 +94,7 @@ def test_get_default_queue(self): message = Message() - self.assertEqual('default_pull', message.get_queue()) + self.assertEqual('default-pull', message.get_queue()) def test_get_task_args(self): """Ensure get_task_args returns the message task_args.""" @@ -90,14 +115,18 @@ def test_get_empty_task_args(self): self.assertEqual({}, message.get_task_args()) - def test_to_dict(self): + @patch('furious.batcher.uuid.uuid4') + def test_to_dict(self, get_uuid): """Ensure to_dict returns a dictionary representation of the Message. """ from furious.batcher import Message + _id = "id" + get_uuid.return_value.hex = _id + task_args = {'other': 'zzz', 'nested': 1} - options = {'task_args': task_args} + options = {'id': _id, 'task_args': task_args} message = Message(**options.copy()) @@ -109,18 +138,23 @@ def test_from_dict(self): task_args = {'other': 'zzz', 'nested': 1} - options = {'task_args': task_args} + options = {'id': 'id', 'task_args': task_args} message = Message.from_dict(options) self.assertEqual(task_args, message.get_task_args()) + self.assertEqual(message.id, 'id') - def test_reconstitution(self): + @patch('furious.batcher.uuid.uuid4') + def test_reconstitution(self, get_uuid): """Ensure to_dict(job.from_dict()) returns the same thing.""" from furious.batcher import Message + _id = "id" + get_uuid.return_value.hex = _id + task_args = {'other': 'zzz', 'nested': 1} - options = {'task_args': task_args} + options = {'id': _id, 'task_args': task_args} message = Message.from_dict(options) @@ -194,6 +228,21 @@ def test_insert(self, queue_mock): class MessageProcessorTestCase(unittest.TestCase): + def setUp(self): + super(MessageProcessorTestCase, self).setUp() + + import os + import uuid + + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + def tearDown(self): + super(MessageProcessorTestCase, self).tearDown() + + import os + + del os.environ['REQUEST_ID_HASH'] + @patch('furious.batcher.time') @patch('furious.batcher.memcache') def test_to_task_with_no_name_passed_in(self, memcache, time): @@ -285,38 +334,41 @@ def test_to_task_with_tag_not_passed_in(self, memcache, time): memcache.get.assert_called_once_with('agg-batch-processor') + @patch('google.appengine.api.taskqueue.TaskRetryOptions', autospec=True) @patch('google.appengine.api.taskqueue.Task', autospec=True) @patch('furious.batcher.time') @patch('furious.batcher.memcache') - def test_to_task_has_correct_arguments(self, memcache, time, task): + def test_to_task_has_correct_arguments(self, memcache, time, task, + task_retry): """Ensure that if no name is passed into the MessageProcessor that it creates a default unique name when creating the task. """ + from furious.async import MAX_RESTARTS from furious.batcher import MessageProcessor memcache.get.return_value = 'current-batch' time.time.return_value = 100 - processor = MessageProcessor('something', queue='test_queue') + task_retry_object = Mock() + task_retry.return_value = task_retry_object + + processor = MessageProcessor('something', queue='test_queue', + id='someid', parent_id='parentid', + context_id="contextid") processor.to_task() task_args = { - 'url': '/_ah/queue/async/something', - 'headers': {}, - 'payload': json.dumps({ - 'queue': 'test_queue', - 'job': ["something", None, None], - 'task_args': { - 'countdown': 30, - 'name': 'processor-processor-current-batch-3' - }, - }), + 'name': 'processor-processor-current-batch-3', + 'url': '/_queue/async/something', 'countdown': 30, - 'name': 'processor-processor-current-batch-3' + 'headers': {}, + 'retry_options': task_retry_object, + 'payload': json.dumps(processor.to_dict()) } task.assert_called_once_with(**task_args) + task_retry.assert_called_once_with(task_retry_limit=MAX_RESTARTS) @patch('furious.batcher.memcache') def test_curent_batch_key_exists_in_cache(self, cache): @@ -353,3 +405,162 @@ def test_curent_batch_key_doesnt_exist_in_cache(self, cache): cache.get.assert_called_once_with('agg-batch-processor') cache.add.assert_called_once_with('agg-batch-processor', 1) + + +class BumpBatchTestCase(unittest.TestCase): + + @patch('furious.batcher.memcache') + def test_cache_incremented_by_key(self, cache): + """Ensure that the cache object is incremented by the key passed in.""" + from furious.batcher import bump_batch + + cache.incr.return_value = 2 + + val = bump_batch('group') + + self.assertEqual(val, 2) + + cache.incr.assert_called_once_with('agg-batch-group') + + +class MessageIteratorTestCase(unittest.TestCase): + + def test_raise_stopiteration_if_no_messages(self): + """Ensure MessageIterator raises StopIteration if no messages.""" + from furious.batcher import MessageIterator + + iterator = MessageIterator('tag', 'qn', 1) + with patch.object(iterator, 'queue') as queue: + queue.lease_tasks_by_tag.return_value = [] + + self.assertRaises(StopIteration, iterator.next) + + def test_iterates(self): + """Ensure MessageIterator instances iterate in loop.""" + from furious.batcher import MessageIterator + + payload = '["test"]' + task = Mock(payload=payload, tag='tag') + task1 = Mock(payload=payload, tag='tag') + task2 = Mock(payload=payload, tag='tag') + + iterator = MessageIterator('tag', 'qn', 1) + + with patch.object(iterator, 'queue') as queue: + queue.lease_tasks_by_tag.return_value = [task, task1, task2] + + results = [payload for payload in iterator] + + self.assertEqual(results, [payload, payload, payload]) + + def test_calls_lease_exactly_once(self): + """Ensure MessageIterator calls lease only once.""" + from furious.batcher import MessageIterator + + payload = '["test"]' + task = Mock(payload=payload, tag='tag') + + message_iterator = MessageIterator('tag', 'qn', 1) + + with patch.object(message_iterator, 'queue') as queue: + queue.lease_tasks_by_tag.return_value = [task] + + iterator = iter(message_iterator) + iterator.next() + + self.assertRaises(StopIteration, iterator.next) + self.assertRaises(StopIteration, iterator.next) + + queue.lease_tasks_by_tag.assert_called_once_with( + 60, 1, tag='tag', deadline=10) + + def test_rerun_after_depletion_calls_once(self): + """Ensure MessageIterator works when used manually.""" + from furious.batcher import MessageIterator + + payload = '["test"]' + task = Mock(payload=payload, tag='tag') + + iterator = MessageIterator('tag', 'qn', 1) + + with patch.object(iterator, 'queue') as queue: + queue.lease_tasks_by_tag.return_value = [task] + + results = [payload for payload in iterator] + self.assertEqual(results, [payload]) + + results = [payload for payload in iterator] + + queue.lease_tasks_by_tag.assert_called_once_with( + 60, 1, tag='tag', deadline=10) + + def test_rerun_after_depletion_doesnt_delete_too_much(self): + """Ensure MessageIterator works when used manually.""" + from furious.batcher import MessageIterator + + payload = '["test"]' + task = Mock(payload=payload, tag='tag') + + iterator = MessageIterator('tag', 'qn', 1, auto_delete=False) + + with patch.object(iterator, 'queue') as queue: + queue.lease_tasks_by_tag.return_value = [task] + + results = [payload for payload in iterator] + self.assertEqual(results, [payload]) + + # This new work should never be leased, but simulates new pending + # work. + task_1 = Mock(payload='["task_1"]', tag='tag') + queue.lease_tasks_by_tag.return_value = [task_1] + + # Iterating again should return the "originally leased" work, not + # new work. + results = [payload for payload in iterator] + self.assertEqual(results, [payload]) + + # Lease should only have been called a single time. + queue.lease_tasks_by_tag.assert_called_once_with( + 60, 1, tag='tag', deadline=10) + + # The delete call should delete only the original work. + iterator.delete_messages() + queue.delete_tasks.assert_called_once_with([task]) + + def test_custom_deadline(self): + """Ensure that a custom deadline gets passed to lease_tasks.""" + from furious.batcher import MessageIterator + + payload = '["test"]' + task = Mock(payload=payload, tag='tag') + + message_iterator = MessageIterator('tag', 'qn', 1, deadline=2) + + with patch.object(message_iterator, 'queue') as queue: + queue.lease_tasks_by_tag.return_value = [task] + + iterator = iter(message_iterator) + iterator.next() + + self.assertRaises(StopIteration, iterator.next) + + queue.lease_tasks_by_tag.assert_called_once_with( + 60, 1, tag='tag', deadline=2) + + @patch('time.time') + def test_time_check(self, time): + """Ensure that a DeadlineExceededError is thrown when the lease takes + over (deadline-0.1) secs. + """ + from google.appengine.runtime import apiproxy_errors + from furious.batcher import MessageIterator + + time.side_effect = [0.0, 9.9] + message_iterator = MessageIterator('tag', 'qn', 1) + + with patch.object(message_iterator, 'queue') as queue: + queue.lease_tasks_by_tag.return_value = [] + + self.assertRaises( + apiproxy_errors.DeadlineExceededError, iter, message_iterator) + diff --git a/furious/tests/test_config.py b/furious/tests/test_config.py new file mode 100644 index 0000000..62e1903 --- /dev/null +++ b/furious/tests/test_config.py @@ -0,0 +1,159 @@ +# +# Copyright 2013 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest +import os +from mock import patch + + +class TestConfigurationLoading(unittest.TestCase): + def setUp(self): + super(TestConfigurationLoading, self).setUp() + self._reset_config() + + def tearDown(self): + super(TestConfigurationLoading, self).tearDown() + self._reset_config() + + def _reset_config(self): + """Reset config.""" + from furious.config import get_config + from furious.config import default_config + + config = get_config() + + get_config().clear() + config.update(default_config()) + + def test_completion_config(self): + + from furious.config import get_completion_cleanup_delay + from furious.config import get_completion_cleanup_queue + from furious.config import get_completion_default_queue + + expected = 'default' + queue = get_completion_cleanup_queue() + self.assertEqual(queue, expected) + + queue = get_completion_default_queue() + self.assertEqual(queue, expected) + + expected = 7600 + delay = get_completion_cleanup_delay() + self.assertEqual(delay, expected) + + def test_load_yaml_config(self): + """Ensure _load_yaml_config will load a specified path.""" + from furious.config import _load_yaml_config + + contents = _load_yaml_config(os.path.join('furious', '_furious.yaml')) + + e = 'persistence: ndb\ncleanupqueue: low-priority\ncleanupdelay: 7600\ndefaultqueue: default\n' + + self.assertEqual(contents, e) + + @patch('os.path.exists', autospec=True) + def test_not_find_yaml(self, mock_exists): + """Ensure when no furious.yaml exists, no file is found.""" + mock_exists.return_value = False + + from furious.config import find_furious_yaml + + config_yaml_path = find_furious_yaml() + + self.assertIsNone(config_yaml_path) + + def test_get_config(self): + """Ensure a config contents produces the expected dictionary.""" + from furious.config import _parse_yaml_config + + example_yaml = str('secret_key: "blah"\n' + 'persistence: bubble\n' + 'task_system: flah\n') + + my_config = _parse_yaml_config(example_yaml) + + self.assertEqual(my_config, {'secret_key': 'blah', + 'persistence': 'bubble', + 'task_system': 'flah', + 'cleanupqueue': 'default', + 'cleanupdelay': 7600, + 'defaultqueue': 'default'}) + + def test_get_configured_persistence_exists(self): + """Ensure a chosen persistence module is selected.""" + from furious.config import _parse_yaml_config + + example_yaml = str('secret_key: "blah"\n' + 'persistence: bubble\n' + 'task_system: flah\n') + + my_config = _parse_yaml_config(example_yaml) + + from furious import config + + config._config = my_config + + persistence_module = config.get_default_persistence_engine( + {'bubble': 'furious.config'}) + + self.assertEqual(persistence_module, config) + + def test_get_config_invalid_yaml(self): + """Ensure an invalid yaml file will raise InvalidYamlFile.""" + from furious.config import InvalidYamlFile + from furious.config import _parse_yaml_config + + example_yaml = str('secret_key:"blah"\n' + 'persistence:bubble\n' + 'task_system:flah\n') + + self.assertRaises(InvalidYamlFile, _parse_yaml_config, example_yaml) + + def test_get_configured_module_by_path(self): + """Ensure _get_configured_module loads options by path.""" + from furious.config import _get_configured_module + from furious import config + + config.get_config()['test_option'] = 'furious.config' + + module = _get_configured_module('test_option') + + self.assertEqual(module, config) + + def test_get_configured_module_by_name(self): + """Ensure _get_configured_module loads options by name.""" + from furious.config import _get_configured_module + from furious import async + from furious import config + + known_modules = {'cfg': 'furious.async'} + + config.get_config()['other_option'] = 'cfg' + + module = _get_configured_module('other_option', known_modules) + + self.assertEqual(module, async) + + def test_get_config_empty_yaml(self): + """Ensure an empty furious.yaml will produce a default config.""" + from furious.config import default_config + from furious.config import _parse_yaml_config + + example_yaml = str('') + + my_config = _parse_yaml_config(example_yaml) + + self.assertEqual(my_config, default_config()) diff --git a/furious/tests/test_job_utils.py b/furious/tests/test_job_utils.py index 51079e2..749b2b7 100644 --- a/furious/tests/test_job_utils.py +++ b/furious/tests/test_job_utils.py @@ -46,26 +46,26 @@ def test_valid_name(self): def test_bad_function_path(self): """Ensure get_function_path_and_options function raises - BadFunctionPathError when given a bad path. + BadObjectPathError when given a bad path. """ - from furious.job_utils import BadFunctionPathError + from furious.errors import BadObjectPathError from furious.job_utils import get_function_path_and_options bad_names = ['', '0abc', 'test.0abc', 'test.ab-cd', 'bad%ness', '.nogood'] for bad_name in bad_names: self.assertRaises( - BadFunctionPathError, get_function_path_and_options, bad_name) + BadObjectPathError, get_function_path_and_options, bad_name) def test_none_as_function_path(self): - """Ensure get_function_path_and_options raises BadFunctionPathError + """Ensure get_function_path_and_options raises BadObjectPathError on missing path. """ - from furious.job_utils import BadFunctionPathError + from furious.errors import BadObjectPathError from furious.job_utils import get_function_path_and_options self.assertRaises( - BadFunctionPathError, get_function_path_and_options, None) + BadObjectPathError, get_function_path_and_options, None) def test_gets_callable_path(self): """Ensure check job function returns the path of a callable.""" @@ -151,8 +151,8 @@ def test_gets_default_options_from_path(self): self.assertEqual(default_options, options) def test_damaged_method_raises(self): - """Ensure a broken mehtod raises BadFunctionPathError.""" - from furious.job_utils import BadFunctionPathError + """Ensure a broken mehtod raises BadObjectPathError.""" + from furious.errors import BadObjectPathError from furious.job_utils import get_function_path_and_options class FakeFunk(object): @@ -162,59 +162,83 @@ def __call__(): some_method = FakeFunk() self.assertRaisesRegexp( - BadFunctionPathError, "Unable to determine path to callable.", + BadObjectPathError, "Invalid object type.", get_function_path_and_options, some_method) -class TestFunctionPathToReference(unittest.TestCase): - """Test that function_path_to_reference finds and load functions.""" +# TODO: Most of the tests from TestGetFunctionPathAndOptions should probably +# be moved into this class. +class TestReferenceToPath(unittest.TestCase): + """Test that reference_to_path converts a reference to a string.""" + + def test_gets_class(self): + """Ensure that reference_to_path can get the path of a class.""" + from furious.job_utils import reference_to_path + + path = reference_to_path(ThrowAway) + + self.assertEqual('furious.tests.test_job_utils.ThrowAway', path) + + +class TestPathToReference(unittest.TestCase): + """Test that path_to_reference finds and load functions.""" @patch('__builtin__.dir') def test_runs_builtin(self, dir_mock): """Ensure builtins are able to be loaded and correctly run.""" - from furious.job_utils import function_path_to_reference + from furious.job_utils import path_to_reference - function = function_path_to_reference("dir") + function = path_to_reference("dir") self.assertIs(dir_mock, function) def test_runs_classmethod(self): """Ensure classmethods are able to be loaded and correctly run.""" - from furious.job_utils import function_path_to_reference + from furious.job_utils import path_to_reference ThrowAway.i_was_ran = False - function = function_path_to_reference( - 'furious.tests.test_job_utils.' - 'ThrowAway.run_me') + function = path_to_reference( + 'furious.tests.test_job_utils.ThrowAway.run_me') function() self.assertTrue(ThrowAway.i_was_ran) def test_raises_on_bogus_builtin(self): """Ensure bad "builins" raise an exception.""" - from furious.job_utils import function_path_to_reference - from furious.job_utils import BadFunctionPathError + from furious.job_utils import path_to_reference + from furious.errors import BadObjectPathError self.assertRaisesRegexp( - BadFunctionPathError, "Unable to find function", - function_path_to_reference, "something_made_up") + BadObjectPathError, "Unable to find function", + path_to_reference, "something_made_up") @patch('email.parser.Parser') def test_runs_std_imported(self, parser_mock): """Ensure run_job is able to correctly run bundled python functions.""" - from furious.job_utils import function_path_to_reference + from furious.job_utils import path_to_reference - function = function_path_to_reference("email.parser.Parser") + function = path_to_reference("email.parser.Parser") self.assertIs(parser_mock, function) def test_raises_on_bogus_std_imported(self): """Ensure run_job raises an exception on bogus standard import.""" - from furious.job_utils import function_path_to_reference - from furious.job_utils import BadFunctionPathError + from furious.job_utils import path_to_reference + from furious.errors import BadObjectPathError self.assertRaisesRegexp( - BadFunctionPathError, "Unable to find function", - function_path_to_reference, "email.parser.NonExistentThing") + BadObjectPathError, "Unable to find function", + path_to_reference, "email.parser.NonExistentThing") + + def test_casts_unicode_name_to_str(self): + """Ensure unicode module_paths do not cause an error.""" + from furious.job_utils import path_to_reference + + imported_module = path_to_reference( + u'furious.tests.dummy_module.dumb') + + from furious.tests.dummy_module import dumb + + self.assertIs(dumb, imported_module) diff --git a/furious/tests/test_processors.py b/furious/tests/test_processors.py index 6c2ced0..e051b67 100644 --- a/furious/tests/test_processors.py +++ b/furious/tests/test_processors.py @@ -88,7 +88,7 @@ def test_runs_with_non_arg_and_kwarg(self, dir_mock): def test_raises_on_missing_job(self): """Ensure run_job raises an exception on bogus standard import.""" from furious.async import Async - from furious.context import NotInContextError + from furious.errors import NotInContextError from furious.processors import run_job work = Async("nothere") @@ -111,7 +111,7 @@ def test_handles_job_exception(self): with _ExecutionContext(work): self.assertRaises(TypeError, run_job) - self.assertIsInstance(work.result, AsyncException) + self.assertIsInstance(work.result.payload, AsyncException) def test_calls_success_callback(self): """Ensure run_job calls the success callback after a successful run.""" @@ -209,6 +209,291 @@ def test_starts_callback_returned_async(self): returned_async.start.assert_called_once_with() + @patch('__builtin__.dir') + def test_AbortAndRestart(self, dir_mock): + """Ensures when AbortAndRestart is raised the Async restarts.""" + from furious.async import Async + from furious.context._execution import _ExecutionContext + from furious.errors import AbortAndRestart + from furious.processors import run_job + + dir_mock.side_effect = AbortAndRestart + mock_success = Mock() + mock_error = Mock() + + work = Async(target='dir', + callbacks={'success': mock_success, + 'error': mock_error}) + + with _ExecutionContext(work): + self.assertRaises(AbortAndRestart, run_job) + + self.assertFalse(mock_success.called) + self.assertFalse(mock_error.called) + + @patch('furious.async.Async.start', autospec=True) + @patch('__builtin__.dir') + def test_Abort(self, dir_mock, mock_start): + """Ensures that when Abort is raised, the Async immediately stops.""" + import logging + + from furious.async import Async + from furious.context._execution import _ExecutionContext + from furious.errors import Abort + from furious.processors import run_job + + class AbortLogHandler(logging.Handler): + + def emit(self, record): + if record.levelno >= logging.ERROR: + raise Exception('An Error level log should not be output') + + logging.getLogger().addHandler(AbortLogHandler()) + + dir_mock.side_effect = Abort + + mock_success = Mock() + mock_error = Mock() + + work = Async(target='dir', + callbacks={'success': mock_success, + 'error': mock_error}) + + with _ExecutionContext(work): + run_job() + + self.assertFalse(mock_success.called) + self.assertFalse(mock_error.called) + self.assertFalse(mock_start.called) + + logging.getLogger().removeHandler(AbortLogHandler()) + + @patch('__builtin__.dir') + def test_BaseException(self, dir_mock): + """Ensure exceptions inheriting from BaseException, such as + DeadlineExceededError, trigger the error handler. + """ + + from furious.async import Async + from furious.context._execution import _ExecutionContext + from furious.processors import run_job + from google.appengine.runtime import DeadlineExceededError + + dir_mock.side_effect = DeadlineExceededError + + mock_success = Mock() + mock_error = Mock() + + work = Async(target='dir', + callbacks={'success': mock_success, + 'error': mock_error}) + + with _ExecutionContext(work): + run_job() + + self.assertFalse(mock_success.called) + self.assertEqual(mock_error.call_count, 1) + + +class TestHandleResults(unittest.TestCase): + """Test that _handle_results does the Right Things.""" + + def setUp(self): + import os + import uuid + + # Ensure each test looks like it is in a new request. + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + @patch('furious.processors._process_results') + def test_defaults_to_process_results(self, processor_mock): + """Ensure _handle_results calls _process_results if not given.""" + from furious.processors import _handle_results + + _handle_results({}) + + processor_mock.assert_called_once_with() + + def test_runs_given_function(self): + """Ensure _handle_results calls the given results processor.""" + from furious.processors import _handle_results + + processor = Mock() + + _handle_results({'_process_results': processor}) + + processor.assert_called_once_with() + + def test_runs_returned_async(self): + """Ensure _handle_results runs Async returned by results processor.""" + from furious.async import Async + from furious.processors import _handle_results + + processor = Mock() + processor.return_value = Mock(spec=Async) + + _handle_results({'_process_results': processor}) + + processor.return_value.start.assert_called_once_with() + + def test_starts_returned_context(self): + """Ensure _handle_results starts Context returned by results processor. + """ + from furious.context.context import Context + from furious.processors import _handle_results + + processor = Mock() + processor.return_value = Mock(spec=Context) + + _handle_results({'_process_results': processor}) + + processor.return_value.start.assert_called_once_with() + + +class TestContextCompletionChecker(unittest.TestCase): + """Test that _handle_context_completion_check does the Right Things.""" + + def setUp(self): + import os + import uuid + + # Ensure each test looks like it is in a new request. + os.environ['REQUEST_ID_HASH'] = uuid.uuid4().hex + + def test_no_completion(self): + """Ensure does not fail if there's no completion checker.""" + from furious.async import Async + from furious.processors import _handle_context_completion_check + + async = Async(dir) + + _handle_context_completion_check(async) + + def test_checker_called_with_async(self): + """Ensure checker called with id as argument.""" + from furious.async import Async + from furious.processors import _handle_context_completion_check + + async = Async(dir) + checker = Mock() + async.update_options(_context_checker=checker) + + _handle_context_completion_check(async) + + checker.assert_called_once_with(async) + + @unittest.skip('not yet implemented.') + def test_async_checker_called_with_id(self): + """Ensure an Async checker called with id as argument.""" + from furious.async import Async + from furious.processors import _handle_context_completion_check + + async = Async(dir) + checker = Mock(spec=Async) + async.update_options(_context_checker=checker) + + _handle_context_completion_check(async) + + checker.update_options.assert_called_once_with(args=['someid']) + checker.start.assert_called_once_with() + + # Make sure didn't try to "call" the Async. + self.assertEqual(checker.call_count, 0) + + +@patch('furious.processors.get_current_async') +class ProcessResultsTestCase(unittest.TestCase): + + def test_is_success_with_callback(self, get_current_async): + """Ensure a sucessful process executes the sucess callback.""" + from furious.processors import _process_results + + async = Mock() + success_callback = Mock() + + async.get_callbacks.return_value = { + 'success': success_callback + } + get_current_async.return_value = async + + result = _process_results() + + self.assertEqual(result, success_callback.return_value) + + def test_is_success_with_async_callback(self, get_current_async): + """Ensure a sucessful process executes the sucess callback and returns + an async if it's callback is an async. + """ + from furious.async import Async + from furious.processors import _process_results + + async = Mock(spec=Async) + success_callback = Mock(spec=Async) + + async.get_callbacks.return_value = { + 'success': success_callback + } + get_current_async.return_value = async + + result = _process_results() + + self.assertEqual(result, success_callback.start.return_value) + + def test_is_success_with_no_callback(self, get_current_async): + """Ensure a sucessful process with no success callback returns the + async result payload if one exists. + """ + from furious.async import Async + from furious.processors import _process_results + + async = Mock(spec=Async) + + async.get_callbacks.return_value = {} + get_current_async.return_value = async + + result = _process_results() + + self.assertEqual(result, async.result.payload) + + def test_is_error_with_callback(self, get_current_async): + """Ensure an error process executes the error callback.""" + from furious.async import Async + from furious.processors import AsyncException + from furious.processors import _process_results + + async = Mock(spec=Async) + async.result.payload = AsyncException("", "", "", "") + error_callback = Mock(spec=Async) + + async.get_callbacks.return_value = { + 'error': error_callback + } + get_current_async.return_value = async + + result = _process_results() + + self.assertEqual(result, error_callback.start.return_value) + + def test_is_error_with_no_callback(self, get_current_async): + """Ensure an error process with no callback raises the error.""" + from furious.async import Async + from furious.async import AsyncResult + from furious.processors import encode_exception + from furious.processors import _process_results + + async = Mock(spec=Async) + + try: + raise Exception() + except Exception, e: + async.result = AsyncResult(payload=encode_exception(e), + status=AsyncResult.ERROR) + + async.get_callbacks.return_value = {} + get_current_async.return_value = async + + self.assertRaises(Exception, _process_results) + def _fake_async_returning_target(async_to_return): return async_to_return @@ -217,5 +502,5 @@ def _fake_async_returning_target(async_to_return): def _fake_result_returning_callback(): from furious.context import get_current_async - return get_current_async().result + return get_current_async().result.payload diff --git a/furious/tests/test_stubs/__init__.py b/furious/tests/test_stubs/__init__.py new file mode 100644 index 0000000..f258ce3 --- /dev/null +++ b/furious/tests/test_stubs/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/furious/tests/test_stubs/appengine/__init__.py b/furious/tests/test_stubs/appengine/__init__.py new file mode 100644 index 0000000..f258ce3 --- /dev/null +++ b/furious/tests/test_stubs/appengine/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/furious/tests/test_stubs/appengine/test_queues.py b/furious/tests/test_stubs/appengine/test_queues.py new file mode 100644 index 0000000..978d762 --- /dev/null +++ b/furious/tests/test_stubs/appengine/test_queues.py @@ -0,0 +1,1143 @@ +# +# Copyright 2012 WebFilings, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import base64 +import json +import os +import unittest + +from nose.plugins.attrib import attr + +from mock import call +from mock import Mock +from mock import patch + +from furious.test_stubs.appengine.queues import _fetch_random_task_from_queue +from furious.test_stubs.appengine.queues import run_random +from furious.test_stubs.appengine.queues import _run_random_task_from_queue +from furious.test_stubs.appengine.queues import _is_furious_task + + +class TestExecuteTask(unittest.TestCase): + """Ensure _execute_task runs the tasks.""" + + @patch('time.ctime') + def test_run_task(self, ctime): + """When a task is passed to _execute_task, make sure it is run. + Ensure the task's environment is cleaned up. + """ + + from furious.context import _local + from furious.test_stubs.appengine.queues import _execute_task + + # Create the async_options to call the target, ctime() + async_options = {'job': ('time.ctime', None, None)} + + body = base64.b64encode(json.dumps(async_options)) + url = '/_queue/async' + + task = {'url': url, 'body': body, 'headers': {}} + + _execute_task(task) + + # Make sure our function was called + self.assertTrue(ctime.called) + + # Make sure context cleanup worked + self.assertFalse('REQUEST_ID_HASH' in os.environ) + self.assertFalse(hasattr(_local._local_context, 'registry')) + + @patch('time.strftime', autospec=True) + def test_run_task_with_args_kwargs(self, strftime): + """When a task with args and kwargs is passed to _execute_task, make + sure it is run with those parameters. + Ensure the task's environment is cleaned up. + """ + + from furious.context import _local + from furious.test_stubs.appengine.queues import _execute_task + + # Create the async_options to call the mocked target, strftime(). + # To test args and kwargs, our arguments to the mocked strftime + # won't match the real strftime's expected parameters. + args = [1, 2] + kwargs = {'my_kwarg': 'my_value'} + async_options = {'job': ('time.strftime', + args, kwargs)} + + body = base64.b64encode(json.dumps(async_options)) + url = '/_queue/async' + + task = {'url': url, 'body': body, 'headers': {}} + + _execute_task(task) + + # Make sure our function was called with the right arguments + strftime.assert_called_once_with(*args, **kwargs) + + # Make sure context cleanup worked + self.assertFalse('REQUEST_ID_HASH' in os.environ) + self.assertFalse(hasattr(_local._local_context, 'registry')) + + +class TestRunQueue(unittest.TestCase): + """Ensure tasks from queues are run.""" + + @patch('furious.test_stubs.appengine.queues._execute_task') + def test_run_queue(self, _execute_task): + """When run() is called, ensure tasks are run, and + the queue is flushed to remove run tasks. Also, ensure True + is returned since messages were processed. + """ + + from furious.test_stubs.appengine.queues import run_queue + + queue_service = Mock() + queue_service.GetTasks.return_value = ['task1', 'task2', 'task3'] + + num_processed = run_queue(queue_service, 'default') + + # Expect _execute_task() to be called for each task + expected_call_args_list = [call('task1', None, None), + call('task2', None, None), + call('task3', None, None)] + + self.assertEquals(_execute_task.call_args_list, + expected_call_args_list) + + # Make sure FlushQueue was called once to clear the queue after + # tasks were processed + self.assertEqual(1, queue_service.FlushQueue.call_count) + + # We should have processed tasks, so verify the num processed. + self.assertEqual(3, num_processed) + + @patch('furious.test_stubs.appengine.queues._execute_task') + def test_run_queue_no_tasks(self, _execute_task): + """When run() is called and there are no tasks in the queue, + ensure _execute_task is not called. + Ensure False is returned since no messages were processed. + """ + + from furious.test_stubs.appengine.queues import run_queue + + queue_service = Mock() + queue_service.GetTasks.return_value = [] + + num_processed = run_queue(queue_service, 'default') + + # Expect _execute_task() to not be called since there are no tasks + self.assertFalse(_execute_task.called) + + # We should not have processed any tasks, so verify 0 processed. + self.assertEqual(0, num_processed) + + +class TestRunQueues(unittest.TestCase): + """Ensure tasks from queues are run.""" + + @patch('furious.test_stubs.appengine.queues.run_queue') + def test_run(self, run_queue): + """Ensure all push queues are processed by run(). + Ensure pull queues are skipped. + """ + + from furious.test_stubs.appengine.queues import run + + queue_descs = [ + {'name': 'default', 'mode': 'push', 'bucket_size': 100}, + {'name': 'default-pull', 'mode': 'pull', 'bucket_size': 5}, + {'name': 'another-pull', 'mode': 'pull', 'bucket_size': 5}, + {'name': 'my_queue', 'mode': 'push', 'bucket_size': 100}] + + queue_service = Mock() + queue_service.GetQueues.side_effect = [queue_descs] + + # Simulate that messages are processed from each push queue. + num_in_default = 2 + num_in_my = 1 + # The two zeros are num remaining in the 2nd iteration for each queue. + run_queue.side_effect = [num_in_default, num_in_my, 0, 0] + + run_result = run(queue_service) + + # Expected 'default' and 'my_queue' to be the only queues processed + # since others are pull queues. + expected_call_args_list = [call(queue_service, 'default', None, None, False), + call(queue_service, 'my_queue', None, None, False), + call(queue_service, 'default', None, None, False), + call(queue_service, 'my_queue', None, None, False)] + + # Ensure run_queue processes the push queues. + self.assertEqual(run_queue.call_args_list, expected_call_args_list) + + # Make sure 2 is returned as the number of messages processed. + self.assertEqual(num_in_default + num_in_my, + run_result['tasks_processed']) + self.assertEqual(2, run_result['iterations']) + + @patch('furious.test_stubs.appengine.queues.run_queue') + def test_run_no_messages(self, run_queue): + """Ensure the return value is False when no messages are processed from + the queues. + Ensure all push queues are processed by run(). + Ensure pull queues are skipped. + """ + + from furious.test_stubs.appengine.queues import run + + queue_descs = [ + {'name': 'default', 'mode': 'push', 'bucket_size': 100}, + {'name': 'default-pull', 'mode': 'pull', 'bucket_size': 5}, + {'name': 'my_queue', 'mode': 'push', 'bucket_size': 100}] + + queue_service = Mock() + queue_service.GetQueues.side_effect = [queue_descs] + + # Simulate that there are no messages processed from any queue. + run_queue.return_value = 0 + + run_result = run(queue_service) + + # Expect 'default' and 'my_queue' to be processed since the other one + # is a pull queue. + expected_call_args_list = [call(queue_service, 'default', None, None, False), + call(queue_service, 'my_queue', None, None, False)] + + # Ensure run_queue processes tries to process the push queues. + self.assertEqual(run_queue.call_args_list, + expected_call_args_list) + + # Make sure that 0 is the number of messages processed. + self.assertEqual(0, run_result['tasks_processed']) + self.assertEqual(1, run_result['iterations']) + + @patch('furious.test_stubs.appengine.queues.run_queue') + def test_run_some_queues_with_messages(self, run_queue): + """Ensure that the tasks_processed in the return dict is 5 when the + first queue processes 5 messages and the next queue processes 0. + Ensure all push queues are processed by run(). + Ensure pull queues are skipped. + """ + + from furious.test_stubs.appengine.queues import run + + queue_descs = [ + {'name': 'default', 'mode': 'push', 'bucket_size': 100}, + {'name': 'my_queue', 'mode': 'push', 'bucket_size': 100}] + + queue_service = Mock(GetQueues=Mock(side_effect=[queue_descs])) + + # Simulate that messages were processed from the first push queue, + # but not the second. + run_queue.side_effect = [5, 0, 0, 0] + + run_result = run(queue_service) + + # Expected 'default' and 'my_queue' to be processed. + # They are processed twice each since messages were processed the + # first iteration. + expected_call_args_list = [call(queue_service, 'default', None, None, False), + call(queue_service, 'my_queue', None, None, False), + call(queue_service, 'default', None, None, False), + call(queue_service, 'my_queue', None, None, False)] + + # Ensure run_queue processes the push queues. + self.assertEqual(run_queue.call_args_list, + expected_call_args_list) + + # Make sure that 5 was returned as the number of messages processed. + self.assertEqual(5, run_result['tasks_processed']) + self.assertEqual(2, run_result['iterations']) + + +@attr('slow') +class TestRunQueuesIntegration(unittest.TestCase): + """Ensure tasks from queues are run.""" + + def setUp(self): + from google.appengine.ext import testbed + + self.testbed = testbed.Testbed() + self.testbed.activate() + self.testbed.init_taskqueue_stub(root_path="") + self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) + + self.taskqueue_service = self.testbed.get_stub( + testbed.TASKQUEUE_SERVICE_NAME) + + def tearDown(self): + self.testbed.deactivate() + + @patch('time.ctime') + def test_run(self, ctime): + """Ensure tasks are run when run_queues is called.""" + + from furious.async import Async + from furious.test_stubs.appengine.queues import run as run_queues + + # Enqueue a couple of tasks + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + # Run the tasks in the queue + run_queues(self.taskqueue_service) + + self.assertEqual(2, ctime.call_count) + + @patch('time.ctime') + def test_run_with_retries(self, ctime): + """ + Ensure tasks are retries when they raise an exception. + Ensure 10 retries are made - 11 total calls. + """ + + from furious.async import Async + from furious.test_stubs.appengine.queues import run as run_queues + + # Count the task runs. + global call_count + call_count = 0 + + def task_call(): + """The function our task will call.""" + + num_retries = int(os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT')) + global call_count + # Ensure the num_retries env var is incremented each time. + self.assertEqual(num_retries, call_count) + call_count += 1 + # Raise an Exception to retry until max retries are reached. + raise Exception() + + ctime.side_effect = task_call + + # Enqueue our task that will fail. + async = Async(target='time.ctime') + async.start() + + # Run the tasks in the queue + run_queues(self.taskqueue_service, enable_retries=True) + + # By default app engine will run the task 11 times. 10 retries + # after the # initial run. + self.assertEqual(11, call_count) + + @patch('time.ctime') + @patch('time.asctime') + @patch('time.accept2dyear') + def test_run_with_retries_and_retries_reset(self, accept2dyear, asctime, + ctime): + """ + Ensure tasks retry counts are separate between asyncs. + Ensure tasks retry counts are reset once an Async is successful. + """ + + from furious.async import Async + from furious.test_stubs.appengine.queues import run as run_queues + + # Count the task runs. + self.async1_call_count = 0 + self.async2_call_count = 0 + self.async3_call_count = 0 + self.async1_retries_env = 0 + self.async2_retries_env = 0 + self.async3_retries_env = 0 + + def task_call_task1(): + """The function task1 will call.""" + + int(os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT')) + + self.async1_call_count += 1 + + if self.async1_call_count < 2: + # Fail once. + raise Exception() + + self.async1_retries_env = int( + os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT')) + + def task_call_task3(): + """The function task3 will call.""" + + self.async3_call_count += 1 + + self.async3_retries_env = int( + os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT')) + + def task_call_task2(): + """The function task2 will call.""" + + self.async2_call_count += 1 + + if self.async2_call_count < 3: + # Fail twice. + raise Exception() + + self.async2_retries_env = int( + os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT')) + + async3 = Async(target='time.accept2dyear') + async3.start() + + ctime.side_effect = task_call_task1 + asctime.side_effect = task_call_task2 + accept2dyear.side_effect = task_call_task3 + + # Enqueue our task that will fail. + async1 = Async(target='time.ctime') + async1.start() + + async2 = Async(target='time.asctime') + async2.start() + + # Run the tasks in the queue + run_queues(self.taskqueue_service, enable_retries=True) + + self.assertEqual(self.async1_call_count, 2) + self.assertEqual(self.async2_call_count, 3) + self.assertEqual(self.async3_call_count, 1) + self.assertEqual(self.async1_retries_env, 1) + self.assertEqual(self.async2_retries_env, 2) + self.assertEqual(self.async3_retries_env, 0) + + # Clear + del self.async1_call_count + del self.async2_call_count + del self.async3_call_count + del self.async1_retries_env + del self.async2_retries_env + del self.async3_retries_env + + +@attr('slow') +class TestPurgeTasks(unittest.TestCase): + """Ensure that purge_tasks() clears tasks from queues.""" + + def setUp(self): + from google.appengine.ext import testbed + + self.testbed = testbed.Testbed() + self.testbed.activate() + self.testbed.init_taskqueue_stub(root_path="") + self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) + + self.taskqueue_service = self.testbed.get_stub( + testbed.TASKQUEUE_SERVICE_NAME) + + def tearDown(self): + self.testbed.deactivate() + + @patch('time.ctime') + def test_purge_tasks_with_no_tasks(self, ctime): + """Ensure no errors occur when purging queues containing no tasks. + Ensure the number of tasks cleared is correct. + """ + + from furious.test_stubs.appengine.queues import purge_tasks + + num_cleared = purge_tasks(self.taskqueue_service) + + # Ensure zero tasks were cleared. + self.assertEqual(0, num_cleared) + + # Ensure no tasks were run + self.assertEqual(0, ctime.call_count) + + @patch('time.ctime') + def test_purge_tasks_with_tasks(self, ctime): + """After queues are run, ensure no tasks are left to execute. + Ensure the number of tasks cleared is correct. + """ + + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import run as run_queues + from furious.test_stubs.appengine.queues import purge_tasks + + # Enqueue a couple of tasks + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + Message(queue='default-pull').insert() + + num_cleared = purge_tasks(self.taskqueue_service) + + # Run the tasks to check if tasks remain + run_queues(self.taskqueue_service) + + # Ensure three tasks were cleared, from 'default' and 'default-pull'. + self.assertEqual(3, num_cleared) + + # Ensure no tasks were run + self.assertEqual(0, ctime.call_count) + + @patch('time.ctime') + def test_purge_tasks_with_queue_names_provided(self, ctime): + """When a list of queue_names is provided, ensure purge_tasks() clears + the tasks and none are left to execute. + Ensure the number of tasks cleared is correct. + """ + + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import run as run_queues + from furious.test_stubs.appengine.queues import purge_tasks + + # Enqueue a couple of tasks + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + Message(queue='default-pull').insert() + + num_cleared = purge_tasks(self.taskqueue_service, ['default']) + + # Run the tasks to check if tasks remain + run_queues(self.taskqueue_service) + + # Ensure two tasks from the default queue were cleared. + self.assertEqual(2, num_cleared) + + # Ensure no tasks were run + self.assertEqual(0, ctime.call_count) + + @patch('time.ctime') + def test_purge_tasks_with_string_passed_to_queue_names(self, ctime): + """If a single queue_name is passed to purge_tasks() instead of a list, + ensure that the queue specified is still cleared. + Ensure the number of tasks cleared is correct. + """ + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import run as run_queues + from furious.test_stubs.appengine.queues import purge_tasks + + # Enqueue a couple of tasks + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + # Insert a pull task + Message(queue='default-pull').insert() + + num_cleared = purge_tasks(self.taskqueue_service, 'default') + + # Run the tasks to check if tasks remain + run_queues(self.taskqueue_service) + + # Ensure two tasks from the default queue were cleared. + self.assertEqual(2, num_cleared) + + # Ensure no tasks were run + self.assertEqual(0, ctime.call_count) + + def test_purge_with_nonexistent_queue(self, ): + """If purge is attempted on a queue that does not exist, ensure that an + Exception is raised. + """ + + from furious.test_stubs.appengine.queues import purge_tasks + + self.assertRaises(Exception, purge_tasks, self.taskqueue_service, + 'non-existent-queue') + + +@attr('slow') +class TestNamesFromQueueService(unittest.TestCase): + """Ensure that get_queue_names(), get_pull_queue_names(), and + get_push_queue_names() return the correct names. + """ + + def setUp(self): + from google.appengine.ext import testbed + + self.testbed = testbed.Testbed() + self.testbed.activate() + self.testbed.init_taskqueue_stub(root_path="") + self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) + + self.taskqueue_service = self.testbed.get_stub( + testbed.TASKQUEUE_SERVICE_NAME) + + def tearDown(self): + self.testbed.deactivate() + + def test_get_pull_queue_names(self): + """Ensure the correct pull queue names are returned from + get_pull_queue_names(). + """ + + from furious.test_stubs.appengine.queues import get_pull_queue_names + + names = get_pull_queue_names(self.taskqueue_service) + + self.assertEqual(names, ['default-pull']) + + def test_get_push_queue_names(self): + """Ensure the correct push queue names are returned from + get_push_queue_names(). + """ + + from furious.test_stubs.appengine.queues import get_push_queue_names + + names = get_push_queue_names(self.taskqueue_service) + + self.assertEqual(names, ['default', 'example']) + + def test_get_queue_names(self): + """Ensure the correct queue names are returned from get_queue_names.""" + + from furious.test_stubs.appengine.queues import get_queue_names + + names = get_queue_names(self.taskqueue_service) + + self.assertEqual(names, ['default', 'default-pull', 'example']) + + +@attr('slow') +class TestGetTasks(unittest.TestCase): + """Ensure that get_tasks(), returns the queues' tasks.""" + + def setUp(self): + from google.appengine.ext import testbed + + self.testbed = testbed.Testbed() + self.testbed.activate() + self.testbed.init_taskqueue_stub(root_path="") + self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) + + self.queue_service = self.testbed.get_stub( + testbed.TASKQUEUE_SERVICE_NAME) + + def tearDown(self): + self.testbed.deactivate() + + def test_get_tasks_when_there_are_no_tasks(self): + """Ensure that no tasks are returned from get_tasks() when no tasks + have been added yet. + """ + + from furious.test_stubs.appengine.queues import get_tasks + + task_dict = get_tasks(self.queue_service) + num_tasks = sum([len(task_list) for task_list in task_dict.values()]) + + self.assertEqual(0, num_tasks) + + def test_get_tasks_from_all_queues(self): + """Ensure all tasks are returned from get_tasks().""" + + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import get_tasks + + # Enqueue a couple of tasks + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + # Insert a pull task + Message(queue='default-pull').insert() + + task_dict = get_tasks(self.queue_service) + num_tasks = sum([len(task_list) for task_list in task_dict.values()]) + + self.assertEqual(3, num_tasks) + + def test_get_tasks_when_queue_names_are_specified(self): + """Ensure queues' tasks are returned from get_tasks() when a list of + queue_names are passed as an argument. + """ + + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import get_tasks + + # Enqueue a couple of tasks + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + # Insert a pull task + Message(queue='default-pull').insert() + + task_dict = get_tasks(self.queue_service, ['default']) + num_tasks = sum([len(task_list) for task_list in task_dict.values()]) + + self.assertEqual(2, num_tasks) + + def test_get_tasks_when_queue_name_string_is_passed(self): + """Ensure a queue's tasks are returned from get_tasks() when a + queue_name is passed as a string. + """ + + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import get_tasks + + # Enqueue a couple of tasks + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + # Insert a pull task + Message(queue='default-pull').insert() + + task_dict = get_tasks(self.queue_service, 'default-pull') + num_tasks = sum([len(task_list) for task_list in task_dict.values()]) + + self.assertEqual(1, num_tasks) + + def test_get_tasks_with_nonexistent_queue(self): + """If a non-existing queue is passed to get_tasks(), ensure that an + Exception is raised. + """ + from furious.test_stubs.appengine.queues import get_tasks + + self.assertRaises(Exception, get_tasks, self.queue_service, + 'non-existent-queue') + + +@attr('slow') +class TestAddTasks(unittest.TestCase): + """Ensure that add_tasks(), adds tasks to App Engine's queues.""" + + def setUp(self): + from google.appengine.ext import testbed + + self.testbed = testbed.Testbed() + self.testbed.activate() + self.testbed.init_taskqueue_stub(root_path="") + self.testbed.get_stub(testbed.TASKQUEUE_SERVICE_NAME) + + self.queue_service = self.testbed.get_stub( + testbed.TASKQUEUE_SERVICE_NAME) + + def tearDown(self): + self.testbed.deactivate() + + def test_add_tasks_when_there_are_no_tasks(self): + """Ensure that no tasks are added to add_tasks() when the + task_dict is empty. + """ + + from furious.test_stubs.appengine.queues import add_tasks + from furious.test_stubs.appengine.queues import purge_tasks + + task_dict = {} + num_added = add_tasks(self.queue_service, task_dict) + + # Purge tasks to count if any tasks remained. + num_purged = purge_tasks(self.queue_service) + + self.assertEqual(0, num_added) + self.assertEqual(0, num_purged) + + @patch('google.appengine.api.taskqueue.Queue.add', autospec=True) + def test_add_tasks_with_empty_queues(self, queue_add): + """Ensure qeueue.add() is not called when there are no tasks to queue. + In some cases adding an empty list causes an error in the taskqueue + stub. + """ + + from furious.test_stubs.appengine.queues import add_tasks + from furious.test_stubs.appengine.queues import purge_tasks + + task_dict = {'default': [], 'default-pull': []} + + # Add using empty lists of tasks. + num_added = add_tasks(self.queue_service, task_dict) + + # Purge tasks to verify the count of tasks added. + num_purged = purge_tasks(self.queue_service) + + # Ensure no tasks were added. + self.assertEqual(0, queue_add.call_count) + self.assertEqual(0, num_added) + self.assertEqual(0, num_purged) + + def test_add_push_queue_tasks(self): + """Ensure that push queue tasks can be added with add_tasks().""" + + from furious.async import Async + from furious.test_stubs.appengine.queues import add_tasks + from furious.test_stubs.appengine.queues import get_tasks + from furious.test_stubs.appengine.queues import purge_tasks + + # Add tasks the normal way so we can get them and test readding them + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + + task_dict = get_tasks(self.queue_service) + + # purge current tasks so we can verify how many we add next. + purge_tasks(self.queue_service) + + num_added = add_tasks(self.queue_service, task_dict) + + # Purge tasks to check how many tasks are in the queues + num_queued = purge_tasks(self.queue_service) + + self.assertEqual(2, num_added) + self.assertEqual(2, num_queued) + + def test_add_pull_queue_tasks(self): + """Ensure that pull tasks can be added with add_tasks().""" + + from furious.batcher import Message + from furious.test_stubs.appengine.queues import add_tasks + from furious.test_stubs.appengine.queues import get_tasks + from furious.test_stubs.appengine.queues import purge_tasks + + # Add tasks the normal way so we can get them and test readding them + Message(queue='default-pull').insert() + + task_dict = get_tasks(self.queue_service) + + # purge current tasks so we can verify how many we add next. + purge_tasks(self.queue_service) + + num_added = add_tasks(self.queue_service, task_dict) + + # Purge tasks to check how many tasks are in the queues + num_queued = purge_tasks(self.queue_service) + + self.assertEqual(1, num_added) + self.assertEqual(1, num_queued) + + def test_add_pull_and_push_queue_tasks(self): + """Ensure that push and pull tasks can be added with add_tasks().""" + + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import add_tasks + from furious.test_stubs.appengine.queues import get_tasks + from furious.test_stubs.appengine.queues import purge_tasks + + # Add tasks the normal way so we can get them and test readding them + async = Async(target='time.ctime') + async.start() + async2 = Async(target='time.ctime') + async2.start() + Message(queue='default-pull').insert() + + task_dict = get_tasks(self.queue_service) + + # purge current tasks so we can verify how many we will add next. + purge_tasks(self.queue_service) + + num_added = add_tasks(self.queue_service, task_dict) + + # Purge tasks to check how many tasks are in the queues + num_queued = purge_tasks(self.queue_service) + + self.assertEqual(3, num_added) + self.assertEqual(3, num_queued) + + @patch('time.ctime') + def test_add_async_and_message_tasks(self, ctime): + """Ensure taskqueue.Task() instances from furious Asyncs and Messages + can be added. + """ + + from google.appengine.api import taskqueue + from furious.async import Async + from furious.batcher import Message + from furious.test_stubs.appengine.queues import add_tasks + from furious.test_stubs.appengine.queues import run as run_queues + + # Create asyncs + async = Async(target='time.ctime') + async2 = Async(target='time.ctime') + + # Create a message + options = {'task_args': {'payload': 'abcdefg'}} + message = Message(payload='abc', **options) + message_task = message.to_task() + + task_dict = {'default': [async.to_task(), async2.to_task()], + 'default-pull': [message_task]} + + num_added = add_tasks(self.queue_service, task_dict) + + # Ensure three tasks were added. + self.assertEqual(3, num_added) + + # Run the tasks to make sure they were inserted correctly. + run_queues(self.queue_service) + + # Ensure both push queue tasks were executed. + self.assertEqual(2, ctime.call_count) + + # Lease the pull queue task and make sure it has the correct payload. + tasks = taskqueue.Queue('default-pull').lease_tasks(3600, 100) + returned_task_message = tasks[0] + + # Ensure pull queue task payload is the same as the original. + self.assertEqual(returned_task_message.payload, message_task.payload) + + +class TestRunRandom(unittest.TestCase): + """Tests random processing of task queues.""" + + def setUp(self): + + self.queue_names = [ + {'name': 'a', 'mode': 'push'}, + {'name': 'b', 'mode': 'push'}, + {'name': 'c', 'mode': 'pull'}, + {'name': 'd', 'mode': 'push'}] + + self.test_queues = { + 'a': [{'name': '1'}, {'name': '2'}], + 'b': [{'name': '4'}, {'name': '5'}], + 'c': [{'name': '7'}, {'name': '9'}], + 'd': [{'name': '11'}, {'name': '12'}]} + + def test_run_without_queues(self): + """Ensure that we exit early if there aren't any queues. + """ + queue_service = Mock() + + tasks_ran = run_random(queue_service, None) + + self.assertEqual(0, tasks_ran) + + @patch('random.seed') + @patch('random.randrange') + @patch('furious.test_stubs.appengine.queues._run_random_task_from_queue') + def test_run_with_empty_queues(self, run_task_from_queue, random_range, + random_seed): + """Ensures that we hit all queue names when all queues are empty. + """ + queue_service = Mock() + + random_range.return_value = 1 + run_task_from_queue.return_value = False + + test_seed = 555 + + tasks_ran = run_random(queue_service, self.queue_names, test_seed) + + self.assertEqual(0, tasks_ran) + + random_seed.assert_called_once_with(test_seed) + + self.assertEqual(3, run_task_from_queue.call_count) + + index = 0 + for queue_name in ['b', 'd', 'a']: + call_args = run_task_from_queue.call_args_list[index] + self.assertEqual(call(queue_service, queue_name), call_args) + index += 1 + + @patch('furious.test_stubs.appengine.queues._run_random_task_from_queue') + def test_run_tasks_in_queues(self, run_task_from_queue): + """Ensures that we run all tasks from popuplated queues. + """ + queue_service = Mock() + + run_task_from_queue.side_effect = self._run_side_effect + + tasks_ran = run_random(queue_service, self.queue_names) + + self.assertEqual(6, tasks_ran) + + self.assertIsNotNone(self.test_queues.get('c')) + self.assertEqual(2, len(self.test_queues.get('c'))) + + for queue_name in ['a', 'b', 'd']: + tasks = self.test_queues.get(queue_name) + self.assertIsNotNone(tasks) + self.assertEqual(0, len(tasks)) + + @patch('furious.test_stubs.appengine.queues._run_random_task_from_queue') + def test_run_tasks_in_queues_greater_than_max(self, run_task_from_queue): + """Ensures that we only run as many tasks as the 'max_tasks' + """ + queue_service = Mock() + + run_task_from_queue.side_effect = self._run_side_effect + + tasks_ran = run_random(queue_service, self.queue_names, max_tasks=3) + + self.assertEqual(3, tasks_ran) + + remaining_tasks = 0 + for queue, tasks in self.test_queues.iteritems(): + remaining_tasks += len(tasks) + self.assertEqual(5, remaining_tasks) + + def _run_side_effect(self, service, queue_name): + + mock_tasks = self.test_queues.get(queue_name, []) + + task = None + if mock_tasks: + task = mock_tasks.pop() + + return task + + +class TestRunRandomTaskFromQueue(unittest.TestCase): + """Tests proper processing of tasks through _run_random_task_from_queue""" + + def setUp(self): + + self.test_queue = 'queue-ABC' + self.test_task = 'task-ABC' + + @patch('furious.test_stubs.appengine.queues._fetch_random_task_from_queue') + def test_run_without_task(self, fetch_task): + """Ensure that we don't run a task if fetch returns None. + """ + fetch_task.return_value = None + + queue_service = Mock() + + result = _run_random_task_from_queue(queue_service, self.test_queue) + + self.assertFalse(result) + + fetch_task.assert_called_once_with(queue_service, self.test_queue) + + @patch('furious.test_stubs.appengine.queues._execute_task') + @patch('furious.test_stubs.appengine.queues._fetch_random_task_from_queue') + def test_run_with_task(self, fetch_task, execute_task): + """Ensure that we handle a task run properly. + """ + task = {'name': self.test_task} + fetch_task.return_value = task + + queue_service = Mock() + queue_service.DeleteTask = Mock() + + result = _run_random_task_from_queue(queue_service, self.test_queue) + + self.assertTrue(result) + + fetch_task.assert_called_once_with(queue_service, self.test_queue) + execute_task.assert_called_once_with(task) + queue_service.DeleteTask.assert_called_once_with( + self.test_queue, self.test_task) + + +class TestFetchRandomTaskFromQueue(unittest.TestCase): + """Ensure tasks from queues are run randomly.""" + + def setUp(self): + + self.test_queue = 'queue-ABC' + + def test_fetch_with_no_tasks(self): + """Ensure None is returned when GetTasks returns None. + """ + queue_service = Mock() + queue_service.GetTasks.return_value = None + + result = _fetch_random_task_from_queue(queue_service, self.test_queue) + + self.assertIsNone(result) + + queue_service.GetTasks.assert_called_once_with(self.test_queue) + + @patch('random.choice') + def test_fetch_with_tasks(self, choice): + """Ensure None is returned when GetTasks returns None. + """ + queue_service = Mock() + queue_service.GetTasks.return_value = ['a', 'b', 'c'] + + choice.return_value = 'b' + + result = _fetch_random_task_from_queue(queue_service, self.test_queue) + + self.assertEqual('b', result) + + queue_service.GetTasks.assert_called_once_with(self.test_queue) + + +class IsFuriousTaskTestCase(unittest.TestCase): + + def test_no_furious_url_prefixes(self): + """Ensure if no non_furious_url_prefixes are passed in True is + returned. + """ + task = {} + furious_url_prefixes = None + non_furious_handler = None + + result = _is_furious_task(task, furious_url_prefixes, + non_furious_handler) + + self.assertTrue(result) + + def test_url_not_url_prefixes(self): + """Ensure task url not in non_furious_url_prefixes True is returned.""" + task = { + 'url': '/_queue/async' + } + furious_url_prefixes = ('/_queue/defer',) + non_furious_handler = None + + result = _is_furious_task(task, furious_url_prefixes, + non_furious_handler) + + self.assertTrue(result) + + def test_url_in_url_prefixes_with_no_handler(self): + """Ensure task url not in furious_url_prefixes True is returned but the + handler is not called.""" + task = { + 'url': '/_queue/defer' + } + furious_url_prefixes = ('/_queue/defer',) + non_furious_handler = None + + result = _is_furious_task(task, furious_url_prefixes, + non_furious_handler) + + self.assertFalse(result) + + def test_url_in_url_prefixes_with_handler(self): + """Ensure task url not in furious_url_prefixes True is returned and + the handler is called. + """ + task = { + 'url': '/_queue/defer', + } + furious_url_prefixes = ('/_queue/defer',) + + non_furious_handler = Mock() + + result = _is_furious_task(task, furious_url_prefixes, + non_furious_handler) + + self.assertFalse(result) + non_furious_handler.assert_called_once_with(task) diff --git a/queue.yaml b/queue.yaml index cdc630f..bf3897d 100644 --- a/queue.yaml +++ b/queue.yaml @@ -4,6 +4,10 @@ queue: rate: 100/s bucket_size: 100 +- name: example + rate: 50/s + bucket_size: 50 + # PULL QUEUES - name: default-pull mode: pull diff --git a/requirements.txt b/requirements.txt index b8ada7b..e69de29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +0,0 @@ -nose -rednose -mock -coverage -NoseGunit diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..563718b --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,6 @@ +nose +rednose==0.3 +mock +coverage +NoseGunit +nose-exclude diff --git a/scripts/fetch_gae_sdk.py b/scripts/fetch_gae_sdk.py new file mode 100644 index 0000000..25e2b2c --- /dev/null +++ b/scripts/fetch_gae_sdk.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Retrieved from https://github.com/Google/oauth2client +"""Fetch the most recent GAE SDK and decompress it in the current directory. +Usage: + fetch_gae_sdk.py [] +Current releases are listed here: + https://www.googleapis.com/storage/v1/b/appengine-sdks/o?prefix=featured +""" + +import json +import os +import StringIO +import sys +import urllib2 +import zipfile + +_SDK_URL = ( + 'https://www.googleapis.com/storage/v1/b/appengine-sdks/o?prefix=featured') + + +def get_gae_versions(): + try: + version_info_json = urllib2.urlopen(_SDK_URL).read() + except: + return {} + try: + version_info = json.loads(version_info_json) + except: + return {} + return version_info.get('items', {}) + + +def _version_tuple(v): + version_string = os.path.splitext(v['name'])[0].rpartition('_')[2] + return tuple(int(x) for x in version_string.split('.')) + + +def get_sdk_urls(sdk_versions): + python_releases = [ + v for v in sdk_versions + if v['name'].startswith('featured/google_appengine')] + current_releases = sorted( + python_releases, key=_version_tuple, reverse=True) + return [release['mediaLink'] for release in current_releases] + + +def main(argv): + if len(argv) > 2: + print('Usage: {} []'.format(argv[0])) + return 1 + dest_dir = argv[1] if len(argv) > 1 else '.' + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + + if os.path.exists(os.path.join(dest_dir, 'google_appengine')): + print('GAE SDK already installed at {}, exiting.'.format(dest_dir)) + return 0 + + sdk_versions = get_gae_versions() + if not sdk_versions: + print('Error fetching GAE SDK version info') + return 1 + sdk_urls = get_sdk_urls(sdk_versions) + for sdk_url in sdk_urls: + try: + sdk_contents = StringIO.StringIO(urllib2.urlopen(sdk_url).read()) + break + except: + pass + else: + print('Could not read SDK from any of {}'.format(sdk_urls)) + return 1 + sdk_contents.seek(0) + try: + zip_contents = zipfile.ZipFile(sdk_contents) + zip_contents.extractall(dest_dir) + print('GAE SDK Installed to {}.'.format(dest_dir)) + except: + print('Error extracting SDK contents') + return 1 + +if __name__ == '__main__': + sys.exit(main(sys.argv[:])) diff --git a/setup.cfg b/setup.cfg index 0f236ec..a139717 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,11 @@ rednose=True verbosity=0 with-nosegunit=True +cover-package=furious +cover-branches=True + +exclude-dir=furious/tests/dummy_module +ATTR=!slow + +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c0f8f42 --- /dev/null +++ b/setup.py @@ -0,0 +1,37 @@ +from setuptools import find_packages, setup + + +def get_version(): + import imp + import os + + with open(os.path.join('furious', '_pkg_meta.py'), 'rb') as f: + mod = imp.load_source('_pkg_meta', 'biloba', f) + + return mod.version + + +setup_args = dict( + name='furious', + version=get_version(), + license='Apache', + description='Furious is a lightweight library that wraps Google App Engine' + 'taskqueues to make building dynamic workflows easy.', + author='Robert Kluin', + author_email='robert.kluin@workiva.com', + url='http://github.com/Workiva/furious', + packages=find_packages(exclude=['example']), + download_url = "https://github.com/Workiva/furious/tarball/v1.3.0", + keywords = ['async', 'gae', 'appengine', 'taskqueue'], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + ], +) + +if __name__ == '__main__': + setup(**setup_args)