From e0902923e2e29fac20550017dee6f7eebab69a74 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 03:26:56 -0400 Subject: [PATCH 01/20] Add galfi classifiers. --- cinje/classify.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 cinje/classify.py diff --git a/cinje/classify.py b/cinje/classify.py new file mode 100644 index 0000000..733ea92 --- /dev/null +++ b/cinje/classify.py @@ -0,0 +1,75 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from marrow.dsl.base import Classifier + + +class CinjeScopeClassifier(Classifier): + """Mark and clean up end-of-scope lines. + + Scopes are increased via block translators, decreased via explicit ": end". + """ + + priority = -1010 + + def classify(self, context, line): + text = line.stripped + + if not text: + return + + if text[0] != ':': + return + + if text[1:].strip() != 'end': + return + + line.line = line.stripped = '' + line.tag.add('_end') + context.input.scope -= 1 + + +class CinjeLineClassifier(Classifier): + """Classify lines into three broad groups: text, code, and comments.""" + + priority = -1000 + + def classify(self, context, line): + text = line.stripped + + if not text: + line.tag.add('blank') + line.line = '' # Blank lines are really blank. + return + + if text[0] == ':': + text = line.line = line.stripped = text[1:].strip() # Code in cinje acquires scope through other means. + line.tag.add('code') + + if not text: + line.tag.add('blank') + return + + if text[0] == '@': + line.tag.add('decorator') + return + + identifier, _, body = text.partition(' ') + + if identifier in ('from', 'import'): + line.tag.add('import') + elif identifier in ('def', ): # TODO: Query valid block handlers. + line.tag.add(identifier) + + # TODO: Extract relevant parts as "CommentClassifier", add to cinje namespace. + elif text[0] == '#' and not text.startswith("#{"): + line.line = text # Eliminate extraneous whitespace and match overall scope. + line.tag.add('code') + line.tag.add('comment') + + if 'coding:' in text: + line.tag.add('encoding') + + else: + line.tag.add('text') From 3a37c7f5fab87536da4a4f6f751dd94327b9bc70 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 03:27:52 -0400 Subject: [PATCH 02/20] Naming correction. --- cinje/classify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cinje/classify.py b/cinje/classify.py index 733ea92..2959278 100644 --- a/cinje/classify.py +++ b/cinje/classify.py @@ -62,7 +62,7 @@ def classify(self, context, line): elif identifier in ('def', ): # TODO: Query valid block handlers. line.tag.add(identifier) - # TODO: Extract relevant parts as "CommentClassifier", add to cinje namespace. + # TODO: Extract relevant parts as "GalfiCommentClassifier", add to cinje namespace. elif text[0] == '#' and not text.startswith("#{"): line.line = text # Eliminate extraneous whitespace and match overall scope. line.tag.add('code') From 6ff4f719b60d972c52d6c64dda671c9988bd3c6e Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 03:36:27 -0400 Subject: [PATCH 03/20] Metadata, notably entry_points, updates. --- setup.py | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 1f4881e..44ef770 100755 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ if sys.version_info < (2, 7): raise SystemExit("Python 2.7 or later is required.") -elif sys.version_info > (3, 0) and sys.version_info < (3, 2): - raise SystemExit("CPython 3.3 or Pypy 3 (3.2) or later is required.") +elif sys.version_info > (3, 0) and sys.version_info < (3, 3): + raise SystemExit("CPython 3.3 or compatible Pypy or later is required.") version = description = url = author = author_email = "" # Silence linter warnings. exec(open(os.path.join("cinje", "release.py")).read()) # Actually populate those values. @@ -60,6 +60,7 @@ "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python", @@ -73,26 +74,36 @@ packages = find_packages(exclude=['bench', 'docs', 'example', 'test', 'htmlcov']), include_package_data = True, package_data = {'': ['README.rst', 'LICENSE.txt']}, - namespace_packages = [], + namespace_packages = ['marrow'], zip_safe = True, entry_points = { - 'cinje.translator': [ - # Block Translators - 'function = cinje.block.function:Function', - 'generic = cinje.block.generic:Generic', - 'module = cinje.block.module:Module', - 'using = cinje.block.using:Using', + 'marrow.dsl.cinje': [ + # Cinje Line Classifiers + 'scope = cinje.classify:CinjeScopeClassifier', + 'line = cinje.classify:CinjeLineClassifier', - # Inline Translators - 'blank = cinje.inline.blank:Blank', - 'code = cinje.inline.code:Code', - 'comment = cinje.inline.comment:Comment', - 'flush = cinje.inline.flush:Flush', - 'require = cinje.inline.require:Require', - 'text = cinje.inline.text:Text', - 'use = cinje.inline.use:Use', - 'pragma = cinje.inline.pragma:Pragma', + # Block Transformers + 'module = cinje.block.module:CinjeModuleTransformer', + #'function = cinje.block.function:CinjeFunctionTransformer', + #'generic = cinje.block.generic:Generic', + #'using = cinje.block.using:Using', + + # Inline Transformers + #'blank = cinje.inline.blank:Blank', + #'code = cinje.inline.code:Code', + #'comment = cinje.inline.comment:Comment', # see TODO from cinje.classify + #'flush = cinje.inline.flush:Flush', + #'require = cinje.inline.require:Require', + #'use = cinje.inline.use:Use', + #'pragma = cinje.inline.pragma:Pragma', + #'text = cinje.inline.text:CinjeTextTransformer', + ], + 'marrow.dsl.cinje.html': [ + # 'text = cinje.inline.text:CinjeHTMLTransformer', + ], + 'marrow.dsl.cinje.xml': [ + # 'text = cinje.inline.text:CinjeXMLTransformer', ], }, @@ -103,6 +114,6 @@ tests_require = tests_require, extras_require = { 'development': tests_require + ['pre-commit'], # Development requirements are the testing requirements. - 'safe': ['webob'], # String safety. + 'safe': ['markupsafe'], # String safety. }, ) From 4e40c7114f030a300d86ec2adc42b834a2b22f84 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 03:44:00 -0400 Subject: [PATCH 04/20] Initial decoder stub. --- cinje/decoder.py | 12 ++++++++++ cinje/encoding.py | 56 ----------------------------------------------- setup.py | 13 +++++++---- 3 files changed, 21 insertions(+), 60 deletions(-) create mode 100644 cinje/decoder.py delete mode 100644 cinje/encoding.py diff --git a/cinje/decoder.py b/cinje/decoder.py new file mode 100644 index 0000000..22a7b93 --- /dev/null +++ b/cinje/decoder.py @@ -0,0 +1,12 @@ +# encoding: utf-8 + +from __future__ import unicode_literals + +from marrow.dsl.decoder import GalfiDecoder + + +class CinjeDecoder(GalfiDecoder): + __slots__ = ('_flags') + + FLAGS = { + } diff --git a/cinje/encoding.py b/cinje/encoding.py deleted file mode 100644 index 9271d91..0000000 --- a/cinje/encoding.py +++ /dev/null @@ -1,56 +0,0 @@ -# encoding: utf-8 - -from __future__ import unicode_literals - -import codecs - -from encodings import utf_8 as utf8 - -from .util import StringIO, bytes, str, Context - - -def transform(input): - #__import__('pudb').set_trace() - translator = Context(input) - return '\n'.join(str(i) for i in translator.stream) - - -def cinje_decode(input, errors='strict', final=True): - if not final: return '', 0 - output = transform(bytes(input).decode('utf8', errors)) - return output, len(input) - - -class CinjeIncrementalDecoder(utf8.IncrementalDecoder): - def _buffer_decode(self, input, errors='strict', final=False): - if not final or len(input) == 0: - return '', 0 - - output = transform(bytes(input).decode('utf8', errors)) - - return output, len(input) - - -class CinjeStreamReader(utf8.StreamReader): - def __init__(self, *args, **kw): - codecs.StreamReader.__init__(self, *args, **kw) - self.stream = StringIO(transform(self.stream)) - - -def cinje_search_function(name): - # I have absolutely no idea how to reliably test this scenario, other than artificially. - if name != 'cinje': # pragma: no cover - return None - - return codecs.CodecInfo( - name = 'cinje', - encode = utf8.encode, - decode = cinje_decode, - incrementalencoder = None, # utf8.IncrementalEncoder, - incrementaldecoder = CinjeIncrementalDecoder, # utf8.IncrementalDecoder, - streamreader = CinjeStreamReader, - streamwriter = utf8.StreamWriter - ) - - -codecs.register(cinje_search_function) diff --git a/setup.py b/setup.py index 44ef770..e0f3b92 100755 --- a/setup.py +++ b/setup.py @@ -78,7 +78,11 @@ zip_safe = True, entry_points = { - 'marrow.dsl.cinje': [ + 'marrow.dsl': [ + 'cinje = cinje.decoder:CinjeDecoder', + ], + + 'marrow.dsl.cinje': [ # Core namespace. # Cinje Line Classifiers 'scope = cinje.classify:CinjeScopeClassifier', 'line = cinje.classify:CinjeLineClassifier', @@ -97,12 +101,13 @@ #'require = cinje.inline.require:Require', #'use = cinje.inline.use:Use', #'pragma = cinje.inline.pragma:Pragma', - #'text = cinje.inline.text:CinjeTextTransformer', ], - 'marrow.dsl.cinje.html': [ + + 'marrow.dsl.cinje.html': [ # HTML-specific string safety helpers and adaptions. # 'text = cinje.inline.text:CinjeHTMLTransformer', ], - 'marrow.dsl.cinje.xml': [ + + 'marrow.dsl.cinje.xml': [ # XML-specific string safety helpers and adaptions. # 'text = cinje.inline.text:CinjeXMLTransformer', ], }, From a548ac1e2217ff1e29250ca421bb6fa6daa3536a Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 03:46:43 -0400 Subject: [PATCH 05/20] Galfi module transformer. --- cinje/block/module.py | 137 ++++++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 66 deletions(-) diff --git a/cinje/block/module.py b/cinje/block/module.py index 868b7b7..2ed42d3 100644 --- a/cinje/block/module.py +++ b/cinje/block/module.py @@ -2,85 +2,90 @@ from __future__ import unicode_literals -from zlib import compress -from base64 import b64encode -from collections import deque +from collections import defaultdict as ddict -from ..util import py, Line +from marrow.dsl.block.module import ModuleTransformer +from marrow.dsl.compat import py2, str +from marrow.dsl.core import Line -def red(numbers): - """Encode the deltas to reduce entropy.""" +class CinjeModuleTransformer(ModuleTransformer): + """A cinje module. - line = 0 - deltas = [] + Where the base `ModuleTransformer` class handles line number mapping and `__futures__` imports for Python 2 + environments, this specialization adds template function name tracking, automatic importing of helpers, and + in-development command-line interface `__main__` handler. - for value in numbers: - deltas.append(value - line) - line = value + Flags: - return b64encode(compress(b''.join(chr(i).encode('latin1') for i in deltas))).decode('latin1') - - - -class Module(object): - """Module handler. + * `free` - ensure no runtime dependency on cinje + * `nomap` - do not emit line number mappings + * `raw` - implies `free`; make no effort to sanitize output + * `unbuffered` - utilize unbuffered output; fragments will be yielded as generated, buffer construction prefixes + will not be generated, + + **Note:** The `raw` flag is **insecure**, but blazingly fast -- use with trusted or pre-sanitized input only! + All text replacements are cast to unicode without error handling. + + Inherits: - This is the initial scope, and the highest priority to ensure its processing of the preamble happens first. + * `buffer` - the named collection of buffers + + Tracks: + + * `templates` - a set of registered template names + * `helpers` - a set of declared used helpers, a shortcut for other transformers + * `_imports` - a mapping of packages to the set of objects acquired from within, from parent class + + For reference, the buffers of a module are divided into: + + * `comment' - shbang, encoding declaration, any additional leading comments and whitespace + * `docstring` - the docstring of the module, if present + * `imports` - the initial block of imports, including whitespace + * `prefix` - any code to be inserted between imports and first non-import line + * `module` - the contents of the module proper + * `suffix` - any code to be appended to the module, prior to the line mapping """ - priority = -100 + __slots__ = ('templates', 'helpers') # Additional data tracked by our specialization. - def match(self, context, line): - return 'init' not in context.flag + FLAGS = { # Global processing flags. + 'free', + 'nomap', + 'raw', + 'unbuffered', + } - def __call__(self, context): - input = context.input - - context.flag.add('init') - context.flag.add('buffer') + # Line templates for easy re-use later. + TEMPLATES = Line('__tmpl__ = ["{}"]') + MAIN = Line('if __name__ == "__main__":') + SINGLE = Line('_cli({})', scope=1) + MULTI = Line('_cli({_tmpl: _tmpl_fn for _tmpl, _tmpl_fn in locals().items() if _tmpl in __tmpl__})', scope=1) + + def __init__(self, decoder): + super(CinjeModuleTransformer, self).__init__(decoder) - imported = False + self.templates = set() + self.helpers = {'str'} if py2 else set() + + def egress(self, context): + capable = not context.flag & {'free', 'raw'} - for line in input: - if not line.stripped or line.stripped[0] == '#': - if not line.stripped.startswith('##') and 'coding:' not in line.stripped: - yield line - continue + if self.templates: + suffix = self.suffix + suffix.append('', self.TEMPLATES.format('", "'.join(self.templates))) - input.push(line) # We're out of the preamble, so put that line back and stop. - break - - # After any existing preamble, but before other imports, we inject our own. - - if py == 2: - yield Line(0, 'from __future__ import unicode_literals') - yield Line(0, '') - - yield Line(0, 'import cinje') - yield Line(0, 'from cinje.helpers import escape as _escape, bless as _bless, iterate, xmlargs as _args, _interrupt, _json') - yield Line(0, '') - yield Line(0, '') - yield Line(0, '__tmpl__ = [] # Exported template functions.') - yield Line(0, '') - - for i in context.stream: - yield i - - if context.templates: - yield Line(0, '') - yield Line(0, '__tmpl__.extend(["' + '", "'.join(context.templates) + '"])') - context.templates = [] - - # Snapshot the line number mapping. - mapping = deque(context.mapping) - mapping.reverse() - - yield Line(0, '') - - if __debug__: - yield Line(0, '__mapping__ = [' + ','.join(str(i) for i in mapping) + ']') + if __debug__ and capable: + self.helpers.add('_cli') + suffix.append('', self.MAIN) + + if len(self.templates) == 1: + tmpl, = self.templates + suffix.append(self.SINGLE.format(tmpl)) + else: + suffix.append(self.MULTI) - yield Line(0, '__gzmapping__ = b"' + red(mapping).replace('"', '\"') + '"') + if capable: + self._imports['cinje.helpers'].update(self.helpers) - context.flag.remove('init') + super(CinjeModuleTransformer, self).egress(context) From 30d5a72f5e59f8020e3e4e5308c68a0eee261cc4 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 03:58:12 -0400 Subject: [PATCH 06/20] Galfi function transformer. --- cinje/block/function.py | 192 ++++++++++++++++------------------------ 1 file changed, 77 insertions(+), 115 deletions(-) diff --git a/cinje/block/function.py b/cinje/block/function.py index 1826ce1..32e860e 100644 --- a/cinje/block/function.py +++ b/cinje/block/function.py @@ -1,153 +1,115 @@ # encoding: utf-8 -import re +from __future__ import unicode_literals -from ..util import py, pypy, ensure_buffer +from marrow.dsl.block.function import FunctionTransformer from ..inline.flush import flush_template +from ..util import ensure_buffer +log = __import__('logging').getLogger(__name__) -class Function(object): + +class CinjeFunctionTransformer(FunctionTransformer): """Proces function declarations within templates. + Used to track if the given function is a template function or not, transform the argument list if such optimization + is warranted, and to add the requisite template processing glue suffix. Functions increase scope. + Syntax: : def : end - """ + Inherits: - priority = -50 + * `name` - the name of the function + * `buffer` - the named collection of buffers - # Patterns to search for bare *, *args, or **kwargs declarations. - STARARGS = re.compile(r'(^|,\s*)\*([^*\s,]+|\s*,|$)') - STARSTARARGS = re.compile(r'(^|,\s*)\*\*\S+') + Tracks: - # Automatically add these as keyword-only scope assignments. - OPTIMIZE = ['_escape', '_bless', '_args'] + * `helpers` - helpers utilized within this template function + * `added` - context tags added via annotation + * `removed` - context tags removed via annotation - def match(self, context, line): - """Match code lines using the "def" keyword.""" - return line.kind == 'code' and line.partitioned[0] == 'def' + As a reminder, functions are divided into: - def _optimize(self, context, argspec): - """Inject speedup shortcut bindings into the argument specification for a function. - - This assigns these labels to the local scope, avoiding a cascade through to globals(), saving time. - - This also has some unfortunate side-effects for using these sentinels in argument default values! - """ - - argspec = argspec.strip() - optimization = ", ".join(i + "=" + i for i in self.OPTIMIZE) - split = None - prefix = '' - suffix = '' - - if argspec: - matches = list(self.STARARGS.finditer(argspec)) - - if matches: - split = matches[-1].span()[1] # Inject after, a la "*args>_<", as we're positional-only arguments. - if split != len(argspec): - prefix = ', ' if argspec[split] == ',' else '' - suffix = '' if argspec[split] == ',' else ', ' - - else: # Ok, we can do this a different way… - matches = list(self.STARSTARARGS.finditer(argspec)) - prefix = ', *, ' - suffix = ', ' - if matches: - split = matches[-1].span()[0] # Inject before, a la ">_<**kwargs". We're positional-only arguments. - if split == 0: - prefix = '*, ' - else: - suffix = '' - else: - split = len(argspec) - suffix = '' - - else: - prefix = '*, ' - - if split is None: - return prefix + optimization + suffix + * `decorator` + * `declaration` + * `docstring` + * `prefix` + * `function` + * `suffix` + * `trailer` + """ + + __slots__ = ('helpers', 'added', 'removed') + + def __init__(self, decoder): + super(CinjeFunctionTransformer, self).__init__(decoder) - return argspec[:split] + prefix + optimization + suffix + argspec[split:] + self.helpers = set() # Specific helpers utilized within the function. + self.added = set() # Flags added through annotation. + self.removed = set() # Flags removed through annotation. - def __call__(self, context): - input = context.input + def process_declaration(self, context, declaration): + line, = declaration # Cinje declarations can only be one line... for now. - declaration = input.next() - line = declaration.partitioned[1] # We don't care about the "def". - line, _, annotation = line.rpartition('->') + text, _, annotation = line.line.partition(' ')[2].rpartition('->') - if annotation and not line: # Swap the values back. - line = annotation + if annotation and not text: # Swap the values back. + text = annotation annotation = '' - name, _, line = line.partition(' ') # Split the function name. + name, _, text = text.partition(' ') # Split the function name out. - argspec = line.rstrip() - name = name.strip() + argspec = text.rstrip() + name = self.name = name.strip() annotation = annotation.lstrip() - added_flags = [] - removed_flags = [] + annotation = {'!dirty', '!text', '!using'} | set(i.lower().strip() for i in annotation.split()) + + # TODO: Re-introduce positional named local scoping optimization for non-Pypy runtimes. + # TODO: Generalize flag processing like this into galfi. - if annotation: - for flag in (i.lower().strip() for i in annotation.split()): - if not flag.strip('!'): continue # Handle standalone exclamation marks. + for flag in annotation: + if not flag.strip('!'): continue # Ignore standalone exclamation marks. + + if flag[0] == '!': + flag = flag[1:] - if flag[0] == '!': - flag = flag[1:] - - if flag in context.flag: - context.flag.remove(flag) - removed_flags.append(flag) - - continue + if flag in context: # We do this rather than discard to track. + context.remove(flag) + self.removed.add(flag) - if flag not in context.flag: - context.flag.add(flag) - added_flags.append(flag) - - if py == 3 and not pypy: - argspec = self._optimize(context, argspec) - - # Reconstruct the line. - - line = 'def ' + name + '(' + argspec + '):' - - # yield declaration.clone(line='@cinje.Function.prepare') # This lets us do some work before and after runtime. - yield declaration.clone(line=line) - - context.scope += 1 - - for i in ensure_buffer(context, False): - yield i + continue + + if flag not in context: + context.add(flag) + self.added.add(flag) - for i in context.stream: - yield i + line = line.clone(line='def ' + name + '(' + argspec + '):') - if 'using' in context.flag: # Clean up that we were using things. - context.flag.remove('using') + for line in super(CinjeFunctionTransformer, self).process_declaration(context, [line]): + yield line + + def egress(self, context): + """Code to be executed when exiting the context of a function. - if 'text' in context.flag: - context.templates.append(name) + Always call super() last in any subclasses. + """ - for i in flush_template(context, reconstruct=False): # Handle the final buffer yield if any content was generated. - yield i + if 'dirty' in context: + self.suffix.append(*flush_template(context, reconstruct=False)) - if 'text' in context.flag: - context.flag.remove('text') + if 'text' in context: + self.prefix.append(*ensure_buffer(context, False)) + context.module.templates.add(self.name) - for flag in added_flags: - if flag in context.flag: - context.flag.remove(flag) + if 'using' in context: + self.prefix.append('_using_stack = []') - for flag in removed_flags: - if flag not in context.flag: - context.flag.add(flag) + context.module.helpers.update(self.helpers) - context.scope -= 1 - + # Reset the manipulated flags to their original state. + context.flag.discard(self.added) + context.flag.update(self.removed) From a2908747a2567ce228b43f482becfbd133903983 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:08:41 -0400 Subject: [PATCH 07/20] Additional metadata. --- cinje/decoder.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cinje/decoder.py b/cinje/decoder.py index 22a7b93..746bdaf 100644 --- a/cinje/decoder.py +++ b/cinje/decoder.py @@ -6,7 +6,21 @@ class CinjeDecoder(GalfiDecoder): - __slots__ = ('_flags') + __slots__ = ( + '_flags', # allow flags to be defined + + # additional options + ) + + EXTENSIONS = { # mapping of joined Path.suffixes to the fully qualified encoding to interpret them using + '.cinje': 'cinje', + '.pyhtml': 'cinje.ns-html', + '.pyxml': 'cinje.ns-xml', + } FLAGS = { + 'free', # ensure no runtime dependency on cinje + 'nomap', # do not emit line number mappings + 'raw', # make no effort to sanitize output; implies free + 'unbuffered', # do not construct buffers and instead yield fragments as generated } From 4696d7465467062a6810cd54dcd24643c17e3693 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:09:24 -0400 Subject: [PATCH 08/20] Initial CLI and restructuring. --- cinje/helpers.py | 86 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/cinje/helpers.py b/cinje/helpers.py index 2be0db0..826d3f9 100644 --- a/cinje/helpers.py +++ b/cinje/helpers.py @@ -1,21 +1,95 @@ # encoding: utf-8 -# pragma: no cover +from __future__ import unicode_literals, print_function + +import sys + +from collections import Mapping from json import dumps as _json -from .util import str, iterate, xmlargs, interruptable as _interrupt, Pipe as pipe +from marrow.dsl.compat import str +from .util import Pipe as pipe +from .util import interruptable as _interrupt +from .util import iterate as _iterate +from .util import stream as _stream +from .util import xmlargs as _xmlargs + try: - from markupsafe import Markup as bless, escape_silent as escape + from markupsafe import Markup as _bless, escape_silent as _escape + except ImportError: - bless = str + _bless = str + try: from html import escape as __escape except: from cgi import escape as __escape - def escape(value): + def _escape(value): return __escape(str(value)) -__all__ = ['bless', 'escape', 'iterate', 'xmlargs', '_interrupt', 'pipe'] +def _cmd(template, argv=None): + """Simplified command-line interface for template invocation. + + Positional arguments are supported, as are named keyword arguments in the forms `--key value` or `--key=value`. + Some value interpolation is performed; numeric values will be integerized, and values may be JSON. + """ + + argv = argv or sys.argv + tmpl, arguments = argv[1], argv[2:] + args = [] + kwargs = {} + dumb = False + pending = None + + def process(value): + try: + value = json.loads(value) + except ValueError: + pass + + return value + + for arg in arguments: + if dumb: + args.append(arg) + continue + + if pending: + kwargs[pending] = process(arg) + pending = None + continue + + if arg == '--': + dumb = True + continue + + if not arg.startswith('--'): + args.append(arg) + continue + + if '=' not in arg: + pending = arg + continue + + arg, sep, value = arg.partition('=') + kwargs[arg] = process(value) + + if isinstance(template, Mapping): + if tmpl not in template: + print("Unknown template function: " + tmpl + " ", file=sys.stderr) + print("Hint: execute symlinks to modules containing multiple templates.", file=sys.stderr, end="\n\n") + print("", file=sys.stderr) + print("Known template functions:", file=sys.stderr) + for tmpl in sorted(template): + print("* ", file=sys.stderr) + + sys.exit(1) + + return method_name, args, kwargs + + + +__all__ = ['str', '_bless', '_escape', '_iterate', '_xmlargs', '_interrupt', 'pipe'] From 5900da2d921c34e593ad8ef1efeeef8245b87776 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:09:57 -0400 Subject: [PATCH 09/20] Added jinja2 streaming tests. --- example/benchmark.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/example/benchmark.py b/example/benchmark.py index cfd9ad3..a7492dd 100644 --- a/example/benchmark.py +++ b/example/benchmark.py @@ -144,9 +144,21 @@ def test_wheezy_template(): """)) - def test_jinja2(): + def test_jinja2_render(): return jinja2_template.render(ctx) + def test_jinja2_generate(): + list(jinja2_template.generate(ctx)) + return '' + + def test_jinja2_generate_first(): + return next(jinja2_template.generate(ctx)) + + def test_jinja2_stream(): + jinja2_template.stream(ctx).dump('/tmp/jinja-stream-test.html') + return '' + + # region: tornado From 93dcbfc9ad68f9fafbd6fb885f466e0bb86f2bf5 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:11:46 -0400 Subject: [PATCH 10/20] Add unbuffered example. --- example/bigtable.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/example/bigtable.py b/example/bigtable.py index a301eec..ba37dbe 100644 --- a/example/bigtable.py +++ b/example/bigtable.py @@ -55,6 +55,24 @@ : end +: def bigtable_nobuffer table=table, frequency=100 -> unbuffered + + + : for i, row in enumerate(table) + + : for key, value in row + + : end + : if not (i % frequency) + : flush + : end + + : end +
${ key }#{ value }
+ +: end + + : def bigtable_fancy table=table, frequency=100 From ffa3a0f0008a15cc170f62327f26989a2ab3733b Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:13:07 -0400 Subject: [PATCH 11/20] Removal of namepace package declaration. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e0f3b92..785f3fc 100755 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ packages = find_packages(exclude=['bench', 'docs', 'example', 'test', 'htmlcov']), include_package_data = True, package_data = {'': ['README.rst', 'LICENSE.txt']}, - namespace_packages = ['marrow'], + namespace_packages = [], zip_safe = True, entry_points = { From 1e85a85846869a25a266effbfe42fe718e548312 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:30:43 -0400 Subject: [PATCH 12/20] Additional flag: wsgi --- cinje/decoder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cinje/decoder.py b/cinje/decoder.py index 746bdaf..645234e 100644 --- a/cinje/decoder.py +++ b/cinje/decoder.py @@ -23,4 +23,5 @@ class CinjeDecoder(GalfiDecoder): 'nomap', # do not emit line number mappings 'raw', # make no effort to sanitize output; implies free 'unbuffered', # do not construct buffers and instead yield fragments as generated + 'wsgi', # generate WSGI compatible template functions; incompatible with unbuffered and free } From 7210d09e5012b6c1f2999e1db454a5c6d5acb4fb Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:31:02 -0400 Subject: [PATCH 13/20] We shall never speak of 3.2 again. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 785f3fc..8590965 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", From 2e9d283199714cb56296ea51c76ba0f3f12ab166 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:35:27 -0400 Subject: [PATCH 14/20] TODOs and helper compatibility. --- cinje/block/using.py | 2 ++ cinje/helpers.py | 9 ++++++++- cinje/inline/use.py | 3 ++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cinje/block/using.py b/cinje/block/using.py index 2682a55..8e616ad 100644 --- a/cinje/block/using.py +++ b/cinje/block/using.py @@ -2,6 +2,8 @@ from ..util import Line, ensure_buffer +# TODO: Implement https://github.com/marrow/cinje/issues/20 + class Using(object): priority = 25 diff --git a/cinje/helpers.py b/cinje/helpers.py index 826d3f9..f0dbc20 100644 --- a/cinje/helpers.py +++ b/cinje/helpers.py @@ -91,5 +91,12 @@ def process(value): return method_name, args, kwargs +# Backwards compatibility, to be removed in 2.0. -__all__ = ['str', '_bless', '_escape', '_iterate', '_xmlargs', '_interrupt', 'pipe'] +bless = _bless +escape = _escape +iterate = _iterate +xmlargs = _xmlargs + + +__all__ = ['str', '_bless', '_escape', '_iterate', '_xmlargs', '_interrupt', 'pipe', '_json'] diff --git a/cinje/inline/use.py b/cinje/inline/use.py index 61d415a..19d6682 100644 --- a/cinje/inline/use.py +++ b/cinje/inline/use.py @@ -4,6 +4,8 @@ PREFIX = '_buffer.extend(' if pypy else '__w(' +# TODO: Implement https://github.com/marrow/cinje/issues/20 + class Use(object): """Consume the result of calling another template function, extending the local buffer. @@ -49,4 +51,3 @@ def __call__(self, context): else: yield declaration.clone(line="for _chunk in " + name + "(" + args + "):") yield declaration.clone(line="yield _chunk", scope=context.scope + 1) - From 4c893b4b31215233ea5c904840f23cf6fd7a6feb Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 26 Apr 2017 04:45:01 -0400 Subject: [PATCH 15/20] Metadata update: version bump, entry_points. --- cinje/release.py | 3 +-- setup.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cinje/release.py b/cinje/release.py index 5e535dc..9f9487e 100644 --- a/cinje/release.py +++ b/cinje/release.py @@ -5,11 +5,10 @@ from collections import namedtuple -version_info = namedtuple('version_info', ('major', 'minor', 'micro', 'releaselevel', 'serial'))(1, 1, 0, 'final', 0) +version_info = namedtuple('version_info', ('major', 'minor', 'micro', 'releaselevel', 'serial'))(1, 2, 0, 'alpha', 1) version = ".".join([str(i) for i in version_info[:3]]) + ((version_info.releaselevel[0] + str(version_info.serial)) if version_info.releaselevel != 'final' else '') author = namedtuple('Author', ['name', 'email'])("Alice Bevan-McGregor", 'alice@gothcandy.com') description = "A Pythonic and ultra fast template engine DSL." url = 'https://github.com/marrow/cinje/' - diff --git a/setup.py b/setup.py index 8590965..15c43f2 100755 --- a/setup.py +++ b/setup.py @@ -87,9 +87,10 @@ 'line = cinje.classify:CinjeLineClassifier', # Block Transformers - 'module = cinje.block.module:CinjeModuleTransformer', - #'function = cinje.block.function:CinjeFunctionTransformer', + 'function = cinje.block.function:CinjeFunctionTransformer', #'generic = cinje.block.generic:Generic', + #'iterate = cinje.block.iterate:Iterate', + 'module = cinje.block.module:CinjeModuleTransformer', #'using = cinje.block.using:Using', # Inline Transformers From 04e300743157af5faab1815a47101e8bce8ecdb4 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Thu, 27 Apr 2017 11:49:17 -0400 Subject: [PATCH 16/20] Cleanup, documentation. --- cinje/block/function.py | 10 +++++----- cinje/block/module.py | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cinje/block/function.py b/cinje/block/function.py index 32e860e..035b6a8 100644 --- a/cinje/block/function.py +++ b/cinje/block/function.py @@ -18,7 +18,7 @@ class CinjeFunctionTransformer(FunctionTransformer): Syntax: - : def + : def [ -> flag[, ...]] : end Inherits: @@ -34,10 +34,10 @@ class CinjeFunctionTransformer(FunctionTransformer): As a reminder, functions are divided into: - * `decorator` - * `declaration` - * `docstring` - * `prefix` + * `decorator` - any leading `@decorator` invocations prior to the declaration + * `declaration` - the function declaration itself as transformed by `process_declaration` + * `docstring` - the initial documentation string, if present + * `prefix` - any * `function` * `suffix` * `trailer` diff --git a/cinje/block/module.py b/cinje/block/module.py index 2ed42d3..3315508 100644 --- a/cinje/block/module.py +++ b/cinje/block/module.py @@ -27,6 +27,13 @@ class CinjeModuleTransformer(ModuleTransformer): **Note:** The `raw` flag is **insecure**, but blazingly fast -- use with trusted or pre-sanitized input only! All text replacements are cast to unicode without error handling. + Utilizes flags: + + * free + * nomap + * raw + * unbuffered + Inherits: * `buffer` - the named collection of buffers @@ -49,13 +56,6 @@ class CinjeModuleTransformer(ModuleTransformer): __slots__ = ('templates', 'helpers') # Additional data tracked by our specialization. - FLAGS = { # Global processing flags. - 'free', - 'nomap', - 'raw', - 'unbuffered', - } - # Line templates for easy re-use later. TEMPLATES = Line('__tmpl__ = ["{}"]') MAIN = Line('if __name__ == "__main__":') From 53b5e467c39640cf4ed63fa3dbbf9a835d23a25b Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Fri, 28 Apr 2017 02:45:51 -0400 Subject: [PATCH 17/20] Documentation, updated version pulls from singular "helper" module. --- cinje/block/module.py | 56 +++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/cinje/block/module.py b/cinje/block/module.py index 3315508..fb15326 100644 --- a/cinje/block/module.py +++ b/cinje/block/module.py @@ -16,57 +16,51 @@ class CinjeModuleTransformer(ModuleTransformer): environments, this specialization adds template function name tracking, automatic importing of helpers, and in-development command-line interface `__main__` handler. - Flags: + Because cinje modules are so similar to standard Python modules, we don't actually have much work to do. - * `free` - ensure no runtime dependency on cinje - * `nomap` - do not emit line number mappings - * `raw` - implies `free`; make no effort to sanitize output - * `unbuffered` - utilize unbuffered output; fragments will be yielded as generated, buffer construction prefixes - will not be generated, - - **Note:** The `raw` flag is **insecure**, but blazingly fast -- use with trusted or pre-sanitized input only! - All text replacements are cast to unicode without error handling. + Global processing flags: - Utilizes flags: - - * free - * nomap - * raw - * unbuffered + * `free` - If defined the resulting bytecode will have no runtime dependnecy on cinje itself. + * `nomap` - Define to disable emission of line number mappings; this can speed up translation and reduce resulting + bytecode size at the cost of increased debugging difficulty. + * `raw` - Implies `free`; make no effort to sanitize output. This is **insecure**, but blazingly fast -- use with + trusted or pre-sanitized input only! + * `unbuffered` - utilize unbuffered output; fragments will be yielded as generated, buffer construction prefixes + will not be generated Inherits: - * `buffer` - the named collection of buffers + * `buffer` - The named collection of buffers. Tracks: - * `templates` - a set of registered template names - * `helpers` - a set of declared used helpers, a shortcut for other transformers - * `_imports` - a mapping of packages to the set of objects acquired from within, from parent class + * `templates` - The names of all module scoped template functions, as a set. + * `helpers` - A set of declared used helpers, a shortcut for other transformers. + * `_imports` - A mapping of packages to the set of objects acquired from within, from parent class. For reference, the buffers of a module are divided into: - * `comment' - shbang, encoding declaration, any additional leading comments and whitespace - * `docstring` - the docstring of the module, if present - * `imports` - the initial block of imports, including whitespace - * `prefix` - any code to be inserted between imports and first non-import line - * `module` - the contents of the module proper - * `suffix` - any code to be appended to the module, prior to the line mapping + * `comment' - Shbang, encoding declaration, any additional leading comments and whitespace. + * `docstring` - the docstring of the module, if present. + * `imports` - the initial block of imports, including whitespace. + * `prefix` - Any code to be inserted between imports and first non-import line. + * `module` - The contents of the module proper. + * `suffix` - Any code to be appended to the module, prior to the line mapping. """ __slots__ = ('templates', 'helpers') # Additional data tracked by our specialization. # Line templates for easy re-use later. - TEMPLATES = Line('__tmpl__ = ["{}"]') - MAIN = Line('if __name__ == "__main__":') - SINGLE = Line('_cli({})', scope=1) + TEMPLATES = Line('__tmpl__ = ["{}"]') # Used to record template functions at the module scope. + MAIN = Line('if __name__ == "__main__":') # Used with one of the following. + SINGLE = Line('_cli({})', scope=1) # There is only one template, so this is easy mode vs. the next. MULTI = Line('_cli({_tmpl: _tmpl_fn for _tmpl, _tmpl_fn in locals().items() if _tmpl in __tmpl__})', scope=1) def __init__(self, decoder): super(CinjeModuleTransformer, self).__init__(decoder) - self.templates = set() - self.helpers = {'str'} if py2 else set() + self.templates = set() # The names of all module scoped template functions, as a set. + self.helpers = {'str'} if py2 else set() # Helpers to import def egress(self, context): capable = not context.flag & {'free', 'raw'} @@ -86,6 +80,6 @@ def egress(self, context): suffix.append(self.MULTI) if capable: - self._imports['cinje.helpers'].update(self.helpers) + self._imports['cinje.helper'].update(self.helpers) super(CinjeModuleTransformer, self).egress(context) From 8f8a7f5ef9bc0e49feec87dd08f5cf3c4855955a Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Fri, 28 Apr 2017 03:05:22 -0400 Subject: [PATCH 18/20] Cleanup. --- cinje/block/module.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cinje/block/module.py b/cinje/block/module.py index fb15326..96f12af 100644 --- a/cinje/block/module.py +++ b/cinje/block/module.py @@ -2,10 +2,8 @@ from __future__ import unicode_literals -from collections import defaultdict as ddict - from marrow.dsl.block.module import ModuleTransformer -from marrow.dsl.compat import py2, str +from marrow.dsl.compat import py2 from marrow.dsl.core import Line @@ -57,26 +55,32 @@ class CinjeModuleTransformer(ModuleTransformer): MULTI = Line('_cli({_tmpl: _tmpl_fn for _tmpl, _tmpl_fn in locals().items() if _tmpl in __tmpl__})', scope=1) def __init__(self, decoder): + """Construct a new module scope.""" + super(CinjeModuleTransformer, self).__init__(decoder) self.templates = set() # The names of all module scoped template functions, as a set. self.helpers = {'str'} if py2 else set() # Helpers to import def egress(self, context): - capable = not context.flag & {'free', 'raw'} + """Executed when exiting the buffered module scope, prior to emitting collapsed lines.""" + + capable = not context.flag & {'free', 'raw'} # Able to utilize helpers. if self.templates: suffix = self.suffix - suffix.append('', self.TEMPLATES.format('", "'.join(self.templates))) + + if 'nomap' not in context.flag: # If mappings are enabled. + suffix.append('', self.TEMPLATES.format('", "'.join(self.templates))) if __debug__ and capable: self.helpers.add('_cli') suffix.append('', self.MAIN) - if len(self.templates) == 1: + if len(self.templates) == 1: # Fast path for modules containing a single template function. tmpl, = self.templates suffix.append(self.SINGLE.format(tmpl)) - else: + elif 'nomap' not in context.flag: # This requires the mapping be present. suffix.append(self.MULTI) if capable: From 70fefd466d783bf11b852a0af5cfef49a45405dd Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Thu, 25 May 2017 01:07:44 -0400 Subject: [PATCH 19/20] Stubs. --- test/test_block/functions.py | 0 test/test_block/test_function.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/test_block/functions.py create mode 100644 test/test_block/test_function.py diff --git a/test/test_block/functions.py b/test/test_block/functions.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_block/test_function.py b/test/test_block/test_function.py new file mode 100644 index 0000000..e69de29 From f48c1f66e909caf3b7220da2310d19b4796eacec Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 19 Jul 2017 14:31:18 -0400 Subject: [PATCH 20/20] Added doc stub. --- cinje/inline/args.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 cinje/inline/args.py diff --git a/cinje/inline/args.py b/cinje/inline/args.py new file mode 100644 index 0000000..05c215e --- /dev/null +++ b/cinje/inline/args.py @@ -0,0 +1,9 @@ +"""Support for single-function template modules. + +This is the Python equivalent of spooky action at a distance. When imported, modules utilizing this will swap +themselves in `sys.path` for the template function produced named `template`. + +Syntax: + + : args [] +""" \ No newline at end of file