diff --git a/.gitignore b/.gitignore index a0d82f7..a6bad6e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc *.sublime-project *.sublime-workspace +__pycache__ \ No newline at end of file diff --git a/contrib/asttokens/__init__.py b/contrib/asttokens/__init__.py new file mode 100644 index 0000000..cde4aab --- /dev/null +++ b/contrib/asttokens/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2016 Grist Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module enhances the Python AST tree with token and source code information, sufficent to +detect the source text of each AST node. This is helpful for tools that make source code +transformations. +""" + +from .line_numbers import LineNumbers +from .asttokens import ASTTokens diff --git a/contrib/asttokens/asttokens.py b/contrib/asttokens/asttokens.py new file mode 100644 index 0000000..130f53d --- /dev/null +++ b/contrib/asttokens/asttokens.py @@ -0,0 +1,196 @@ +# Copyright 2016 Grist Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import bisect +import token +import tokenize +import io +import six +from six.moves import xrange # pylint: disable=redefined-builtin +from .line_numbers import LineNumbers +from .util import Token, match_token +from .mark_tokens import MarkTokens + +class ASTTokens(object): + """ + ASTTokens maintains the text of Python code in several forms: as a string, as line numbers, and + as tokens, and is used to mark and access token and position information. + + ``source_text`` must be a unicode or UTF8-encoded string. If you pass in UTF8 bytes, remember + that all offsets you'll get are to the unicode text, which is available as the ``.text`` + property. + + If ``parse`` is set, the ``source_text`` will be parsed with ``ast.parse()``, and the resulting + tree marked with token info and made available as the ``.tree`` property. + + If ``tree`` is given, it will be marked and made available as the ``.tree`` property. In + addition to the trees produced by the ``ast`` module, ASTTokens will also mark trees produced + using ``astroid`` library . + + If only ``source_text`` is given, you may use ``.mark_tokens(tree)`` to mark the nodes of an AST + tree created separately. + """ + def __init__(self, source_text, parse=False, tree=None): + if isinstance(source_text, six.binary_type): + source_text = source_text.decode('utf8') + + self._tree = ast.parse(source_text) if parse else tree + + self._text = source_text + self._line_numbers = LineNumbers(source_text) + + # Tokenize the code. + self._tokens = list(self._generate_tokens(source_text)) + + # Extract the start positions of all tokens, so that we can quickly map positions to tokens. + self._token_offsets = [tok.startpos for tok in self._tokens] + + if self._tree: + self.mark_tokens(self._tree) + + + def mark_tokens(self, root_node): + """ + Given the root of the AST or Astroid tree produced from source_text, visits all nodes marking + them with token and position information by adding ``.first_token`` and + ``.last_token``attributes. This is done automatically in the constructor when ``parse`` or + ``tree`` arguments are set, but may be used manually with a separate AST or Astroid tree. + """ + # The hard work of this class is done by MarkTokens + MarkTokens(self).visit_tree(root_node) + + + def _generate_tokens(self, text): + """ + Generates tokens for the given code. + """ + # This is technically an undocumented API for Python3, but allows us to use the same API as for + # Python2. See http://stackoverflow.com/a/4952291/328565. + for index, tok in enumerate(tokenize.generate_tokens(io.StringIO(text).readline)): + tok_type, tok_str, start, end, line = tok + yield Token(tok_type, tok_str, start, end, line, index, + self._line_numbers.line_to_offset(start[0], start[1]), + self._line_numbers.line_to_offset(end[0], end[1])) + + @property + def text(self): + """The source code passed into the constructor.""" + return self._text + + @property + def tokens(self): + """The list of tokens corresponding to the source code from the constructor.""" + return self._tokens + + @property + def tree(self): + """The root of the AST tree passed into the constructor or parsed from the source code.""" + return self._tree + + def get_token_from_offset(self, offset): + """ + Returns the token containing the given character offset (0-based position in source text), + or the preceeding token if the position is between tokens. + """ + return self._tokens[bisect.bisect(self._token_offsets, offset) - 1] + + def get_token(self, lineno, col_offset): + """ + Returns the token containing the given (lineno, col_offset) position, or the preceeding token + if the position is between tokens. + """ + # TODO: add test for multibyte unicode. We need to translate offsets from ast module (which + # are in utf8) to offsets into the unicode text. tokenize module seems to use unicode offsets + # but isn't explicit. + return self.get_token_from_offset(self._line_numbers.line_to_offset(lineno, col_offset)) + + def get_token_from_utf8(self, lineno, col_offset): + """ + Same as get_token(), but interprets col_offset as a UTF8 offset, which is what `ast` uses. + """ + return self.get_token(lineno, self._line_numbers.from_utf8_col(lineno, col_offset)) + + def next_token(self, tok, include_extra=False): + """ + Returns the next token after the given one. If include_extra is True, includes non-coding + tokens from the tokenize module, such as NL and COMMENT. + """ + i = tok.index + 1 + if not include_extra: + while self._tokens[i].type >= token.N_TOKENS: + i += 1 + return self._tokens[i] + + def prev_token(self, tok, include_extra=False): + """ + Returns the previous token before the given one. If include_extra is True, includes non-coding + tokens from the tokenize module, such as NL and COMMENT. + """ + i = tok.index - 1 + if not include_extra: + while self._tokens[i].type >= token.N_TOKENS: + i -= 1 + return self._tokens[i] + + def find_token(self, start_token, tok_type, tok_str=None, reverse=False): + """ + Looks for the first token, starting at start_token, that matches tok_type and, if given, the + token string. Searches backwards if reverse is True. + """ + t = start_token + advance = self.prev_token if reverse else self.next_token + while not match_token(t, tok_type, tok_str) and not token.ISEOF(t.type): + t = advance(t) + return t + + def token_range(self, first_token, last_token, include_extra=False): + """ + Yields all tokens in order from first_token through and including last_token. If + include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT. + """ + for i in xrange(first_token.index, last_token.index + 1): + if include_extra or self._tokens[i].type < token.N_TOKENS: + yield self._tokens[i] + + def get_tokens(self, node, include_extra=False): + """ + Yields all tokens making up the given node. If include_extra is True, includes non-coding + tokens such as tokenize.NL and .COMMENT. + """ + return self.token_range(node.first_token, node.last_token, include_extra=include_extra) + + def get_text_range(self, node): + """ + After mark_tokens() has been called, returns the (startpos, endpos) positions in source text + corresponding to the given node. Returns (0, 0) for nodes (like `Load`) that don't correspond + to any particular text. + """ + if not hasattr(node, 'first_token'): + return (0, 0) + + start = node.first_token.startpos + if any(match_token(t, token.NEWLINE) for t in self.get_tokens(node)): + # Multi-line nodes would be invalid unless we keep the indentation of the first node. + start = self._text.rfind('\n', 0, start) + 1 + + return (start, node.last_token.endpos) + + def get_text(self, node): + """ + After mark_tokens() has been called, returns the text corresponding to the given node. Returns + '' for nodes (like `Load`) that don't correspond to any particular text. + """ + start, end = self.get_text_range(node) + return self._text[start : end] diff --git a/contrib/asttokens/line_numbers.py b/contrib/asttokens/line_numbers.py new file mode 100644 index 0000000..b91b00f --- /dev/null +++ b/contrib/asttokens/line_numbers.py @@ -0,0 +1,71 @@ +# Copyright 2016 Grist Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bisect +import re + +_line_start_re = re.compile(r'^', re.M) + +class LineNumbers(object): + """ + Class to convert between character offsets in a text string, and pairs (line, column) of 1-based + line and 0-based column numbers, as used by tokens and AST nodes. + + This class expects unicode for input and stores positions in unicode. But it supports + translating to and from utf8 offsets, which are used by ast parsing. + """ + def __init__(self, text): + # A list of character offsets of each line's first character. + self._line_offsets = [m.start(0) for m in _line_start_re.finditer(text)] + self._text = text + self._text_len = len(text) + self._utf8_offset_cache = {} # maps line num to list of char offset for each byte in line + + def from_utf8_col(self, line, utf8_column): + """ + Given a 1-based line number and 0-based utf8 column, returns a 0-based unicode column. + """ + offsets = self._utf8_offset_cache.get(line) + if offsets is None: + end_offset = self._line_offsets[line] if line < len(self._line_offsets) else self._text_len + line_text = self._text[self._line_offsets[line - 1] : end_offset] + + offsets = [i for i,c in enumerate(line_text) for byte in c.encode('utf8')] + offsets.append(len(line_text)) + self._utf8_offset_cache[line] = offsets + + return offsets[max(0, min(len(offsets), utf8_column))] + + def line_to_offset(self, line, column): + """ + Converts 1-based line number and 0-based column to 0-based character offset into text. + """ + line -= 1 + if line >= len(self._line_offsets): + return self._text_len + elif line < 0: + return 0 + else: + return min(self._line_offsets[line] + max(0, column), self._text_len) + + def offset_to_line(self, offset): + """ + Converts 0-based character offset to pair (line, col) of 1-based line and 0-based column + numbers. + """ + offset = max(0, min(self._text_len, offset)) + line_index = bisect.bisect_right(self._line_offsets, offset) - 1 + return (line_index + 1, offset - self._line_offsets[line_index]) + + diff --git a/contrib/asttokens/mark_tokens.py b/contrib/asttokens/mark_tokens.py new file mode 100644 index 0000000..f48fb77 --- /dev/null +++ b/contrib/asttokens/mark_tokens.py @@ -0,0 +1,275 @@ +# Copyright 2016 Grist Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import numbers +import token +from . import util + + +# Mapping of matching braces. To find a token here, look up token[:2]. +_matching_pairs_left = { + (token.OP, '('): (token.OP, ')'), + (token.OP, '['): (token.OP, ']'), + (token.OP, '{'): (token.OP, '}'), +} + +_matching_pairs_right = { + (token.OP, ')'): (token.OP, '('), + (token.OP, ']'): (token.OP, '['), + (token.OP, '}'): (token.OP, '{'), +} + + +class MarkTokens(object): + """ + Helper that visits all nodes in the AST tree and assigns .first_token and .last_token attributes + to each of them. This is the heart of the token-marking logic. + """ + def __init__(self, code): + self._code = code + self._methods = util.NodeMethods() + self._iter_children = None + + def visit_tree(self, node): + self._iter_children = util.iter_children_func(node) + util.visit_tree(node, self._visit_before_children, self._visit_after_children) + + def _visit_before_children(self, node, parent_token): + col = getattr(node, 'col_offset', None) + token = self._code.get_token_from_utf8(node.lineno, col) if col is not None else None + + if not token and util.is_module(node): + # We'll assume that a Module node starts at the start of the source code. + token = self._code.get_token(1, 0) + + # Use our own token, or our parent's if we don't have one, to pass to child calls as + # parent_token argument. The second value becomes the token argument of _visit_after_children. + return (token or parent_token, token) + + def _visit_after_children(self, node, parent_token, token): + # This processes the node generically first, after all children have been processed. + + # Get the first and last tokens that belong to children. Note how this doesn't assume that we + # iterate through children in order that corresponds to occurrence in source code. This + # assumption can fail (e.g. with return annotations). + first = token + last = None + for child in self._iter_children(node): + if not first or child.first_token.index < first.index: + first = child.first_token + if not last or child.last_token.index > last.index: + last = child.last_token + + # If we don't have a first token from _visit_before_children, and there were no children, then + # use the parent's token as the first token. + first = first or parent_token + + # If no children, set last token to the first one. + last = last or first + + # Statements continue to before NEWLINE. This helps cover a few different cases at once. + if util.is_stmt(node): + last = self._find_last_in_line(last) + + # Capture any unmatched brackets. + first, last = self._expand_to_matching_pairs(first, last, node) + + # Give a chance to node-specific methods to adjust. + nfirst, nlast = self._methods.get(self, node.__class__)(node, first, last) + + if (nfirst, nlast) != (first, last): + # If anything changed, expand again to capture any unmatched brackets. + nfirst, nlast = self._expand_to_matching_pairs(nfirst, nlast, node) + + node.first_token = nfirst + node.last_token = nlast + + def _find_last_in_line(self, start_token): + try: + newline = self._code.find_token(start_token, token.NEWLINE) + except IndexError: + newline = self._code.find_token(start_token, token.ENDMARKER) + return self._code.prev_token(newline) + + def _iter_non_child_tokens(self, first_token, last_token, node): + """ + Generates all tokens in [first_token, last_token] range that do not belong to any children of + node. E.g. `foo(bar)` has children `foo` and `bar`, but we would yield the `(`. + """ + tok = first_token + for n in self._iter_children(node): + for t in self._code.token_range(tok, self._code.prev_token(n.first_token)): + yield t + if n.last_token.index >= last_token.index: + return + tok = self._code.next_token(n.last_token) + + for t in self._code.token_range(tok, last_token): + yield t + + def _expand_to_matching_pairs(self, first_token, last_token, node): + """ + Scan tokens in [first_token, last_token] range that are between node's children, and for any + unmatched brackets, adjust first/last tokens to include the closing pair. + """ + # We look for opening parens/braces among non-child tokens (i.e. tokens between our actual + # child nodes). If we find any closing ones, we match them to the opens. + to_match_right = [] + to_match_left = [] + for tok in self._iter_non_child_tokens(first_token, last_token, node): + tok_info = tok[:2] + if to_match_right and tok_info == to_match_right[-1]: + to_match_right.pop() + elif tok_info in _matching_pairs_left: + to_match_right.append(_matching_pairs_left[tok_info]) + elif tok_info in _matching_pairs_right: + to_match_left.append(_matching_pairs_right[tok_info]) + + # Once done, extend `last_token` to match any unclosed parens/braces. + for match in reversed(to_match_right): + last = self._code.next_token(last_token) + # Allow for a trailing comma before the closing delimiter. + if util.match_token(last, token.OP, ','): + last = self._code.next_token(last) + # Now check for the actual closing delimiter. + if util.match_token(last, *match): + last_token = last + + # And extend `first_token` to match any unclosed opening parens/braces. + for match in to_match_left: + first = self._code.prev_token(first_token) + if util.match_token(first, *match): + first_token = first + + return (first_token, last_token) + + #---------------------------------------------------------------------- + # Node visitors. Each takes a preliminary first and last tokens, and returns the adjusted pair + # that will actually be assigned. + + def visit_default(self, node, first_token, last_token): + # pylint: disable=no-self-use + # By default, we don't need to adjust the token we computed earlier. + return (first_token, last_token) + + def handle_comp(self, open_brace, node, first_token, last_token): + # For list/set/dict comprehensions, we only get the token of the first child, so adjust it to + # include the opening brace (the closing brace will be matched automatically). + before = self._code.prev_token(first_token) + util.expect_token(before, token.OP, open_brace) + return (before, last_token) + + def visit_listcomp(self, node, first_token, last_token): + return self.handle_comp('[', node, first_token, last_token) + + if six.PY2: + # We shouldn't do this on PY3 because its SetComp/DictComp already have a correct start. + def visit_setcomp(self, node, first_token, last_token): + return self.handle_comp('{', node, first_token, last_token) + + def visit_dictcomp(self, node, first_token, last_token): + return self.handle_comp('{', node, first_token, last_token) + + def visit_comprehension(self, node, first_token, last_token): + # The 'comprehension' node starts with 'for' but we only get first child; we search backwards + # to find the 'for' keyword. + first = self._code.find_token(first_token, token.NAME, 'for', reverse=True) + return (first, last_token) + + def handle_attr(self, node, first_token, last_token): + # Attribute node has ".attr" (2 tokens) after the last child. + dot = self._code.find_token(last_token, token.OP, '.') + name = self._code.next_token(dot) + util.expect_token(name, token.NAME) + return (first_token, name) + + visit_attribute = handle_attr + visit_assignattr = handle_attr + visit_delattr = handle_attr + + def handle_doc(self, node, first_token, last_token): + # With astroid, nodes that start with a doc-string can have an empty body, in which case we + # need to adjust the last token to include the doc string. + if not node.body and getattr(node, 'doc', None): + last_token = self._code.find_token(last_token, token.STRING) + return (first_token, last_token) + + visit_classdef = handle_doc + visit_funcdef = handle_doc + + def visit_call(self, node, first_token, last_token): + # A function call isn't over until we see a closing paren. Remember that last_token is at the + # end of all children, so we are not worried about encountering a paren that belongs to a + # child. + return (first_token, self._code.find_token(last_token, token.OP, ')')) + + def visit_subscript(self, node, first_token, last_token): + # A subscript operations isn't over until we see a closing bracket. Similar to function calls. + return (first_token, self._code.find_token(last_token, token.OP, ']')) + + def visit_tuple(self, node, first_token, last_token): + # A tuple doesn't include parens; if there is a trailing comma, make it part of the tuple. + try: + maybe_comma = self._code.next_token(last_token) + if util.match_token(maybe_comma, token.OP, ','): + last_token = maybe_comma + except IndexError: + pass + return (first_token, last_token) + + def visit_str(self, node, first_token, last_token): + # Multiple adjacent STRING tokens form a single string. + last = self._code.next_token(last_token) + while util.match_token(last, token.STRING): + last_token = last + last = self._code.next_token(last_token) + return (first_token, last_token) + + def visit_num(self, node, first_token, last_token): + # A constant like '-1' gets turned into two tokens; this will skip the '-'. + while util.match_token(last_token, token.OP): + last_token = self._code.next_token(last_token) + return (first_token, last_token) + + # In Astroid, the Num and Str nodes are replaced by Const. + def visit_const(self, node, first_token, last_token): + if isinstance(node.value, numbers.Number): + return self.visit_num(node, first_token, last_token) + elif isinstance(node.value, six.string_types): + return self.visit_str(node, first_token, last_token) + return (first_token, last_token) + + def visit_keyword(self, node, first_token, last_token): + if node.arg is not None: + equals = self._code.find_token(first_token, token.OP, '=', reverse=True) + name = self._code.prev_token(equals) + util.expect_token(name, token.NAME, node.arg) + first_token = name + return (first_token, last_token) + + def visit_starred(self, node, first_token, last_token): + # Astroid has 'Starred' nodes (for "foo(*bar)" type args), but they need to be adjusted. + if not util.match_token(first_token, token.OP, '*'): + star = self._code.prev_token(first_token) + if util.match_token(star, token.OP, '*'): + first_token = star + return (first_token, last_token) + + def visit_assignname(self, node, first_token, last_token): + # Astroid may turn 'except' clause into AssignName, but we need to adjust it. + if util.match_token(first_token, token.NAME, 'except'): + colon = self._code.find_token(last_token, token.OP, ':') + first_token = last_token = self._code.prev_token(colon) + return (first_token, last_token) diff --git a/contrib/asttokens/util.py b/contrib/asttokens/util.py new file mode 100644 index 0000000..4dd2f27 --- /dev/null +++ b/contrib/asttokens/util.py @@ -0,0 +1,236 @@ +# Copyright 2016 Grist Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import collections +import token +from six import iteritems + + +def token_repr(tok_type, string): + """Returns a human-friendly representation of a token with the given type and string.""" + # repr() prefixes unicode with 'u' on Python2 but not Python3; strip it out for consistency. + return '%s:%s' % (token.tok_name[tok_type], repr(string).lstrip('u')) + + +class Token(collections.namedtuple('Token', 'type string start end line index startpos endpos')): + """ + TokenInfo is an 8-tuple containing the same 5 fields as the tokens produced by the tokenize + module, and 3 additional ones useful for this module: + + - [0] .type Token type (see token.py) + - [1] .string Token (a string) + - [2] .start Starting (row, column) indices of the token (a 2-tuple of ints) + - [3] .end Ending (row, column) indices of the token (a 2-tuple of ints) + - [4] .line Original line (string) + - [5] .index Index of the token in the list of tokens that it belongs to. + - [6] .startpos Starting character offset into the input text. + - [7] .endpos Ending character offset into the input text. + """ + def __str__(self): + return token_repr(self.type, self.string) + + +def match_token(token, tok_type, tok_str=None): + """Returns true if token is of the given type and, if a string is given, has that string.""" + return token.type == tok_type and (tok_str is None or token.string == tok_str) + + +def expect_token(token, tok_type, tok_str=None): + """ + Verifies that the given token is of the expected type. If tok_str is given, the token string + is verified too. If the token doesn't match, raises an informative ValueError. + """ + if not match_token(token, tok_type, tok_str): + raise ValueError("Expected token %s, got %s on line %s col %s" % ( + token_repr(tok_type, tok_str), str(token), + token.start[0], token.start[1] + 1)) + + +def iter_children(node): + """ + Yields all direct children of a AST node, skipping children that are singleton nodes. + """ + return iter_children_astroid(node) if hasattr(node, 'get_children') else iter_children_ast(node) + + +def iter_children_func(node): + """ + Returns a slightly more optimized function to use in place of ``iter_children``, depending on + whether ``node`` is from ``ast`` or from the ``astroid`` module. + """ + return iter_children_astroid if hasattr(node, 'get_children') else iter_children_ast + + +def iter_children_astroid(node): + # Don't attempt to process children of JoinedStr nodes, which we can't fully handle yet. + if is_joined_str(node): + return [] + + return node.get_children() + + +SINGLETONS = {c for n, c in iteritems(ast.__dict__) if isinstance(c, type) and + issubclass(c, (ast.expr_context, ast.boolop, ast.operator, ast.unaryop, ast.cmpop))} + +def iter_children_ast(node): + # Don't attempt to process children of JoinedStr nodes, which we can't fully handle yet. + if is_joined_str(node): + return + + for child in ast.iter_child_nodes(node): + # Skip singleton children; they don't reflect particular positions in the code and break the + # assumptions about the tree consisting of distinct nodes. Note that collecting classes + # beforehand and checking them in a set is faster than using isinstance each time. + if child.__class__ not in SINGLETONS: + yield child + + +stmt_class_names = {n for n, c in iteritems(ast.__dict__) + if isinstance(c, type) and issubclass(c, ast.stmt)} +expr_class_names = ({n for n, c in iteritems(ast.__dict__) + if isinstance(c, type) and issubclass(c, ast.expr)} | + {'AssignName', 'DelName', 'Const', 'AssignAttr', 'DelAttr'}) + +# These feel hacky compared to isinstance() but allow us to work with both ast and astroid nodes +# in the same way, and without even importing astroid. +def is_expr(node): + """Returns whether node is an expression node.""" + return node.__class__.__name__ in expr_class_names + +def is_stmt(node): + """Returns whether node is a statement node.""" + return node.__class__.__name__ in stmt_class_names + +def is_module(node): + """Returns whether node is a module node.""" + return node.__class__.__name__ == 'Module' + +def is_joined_str(node): + """Returns whether node is a JoinedStr node, used to represent f-strings.""" + # At the moment, nodes below JoinedStr have wrong line/col info, and trying to process them only + # leads to errors. + return node.__class__.__name__ == 'JoinedStr' + + +# Sentinel value used by visit_tree(). +_PREVISIT = object() + +def visit_tree(node, previsit, postvisit): + """ + Scans the tree under the node depth-first using an explicit stack. It avoids implicit recursion + via the function call stack to avoid hitting 'maximum recursion depth exceeded' error. + + It calls ``previsit()`` and ``postvisit()`` as follows: + + * ``previsit(node, par_value)`` - should return ``(par_value, value)`` + ``par_value`` is as returned from ``previsit()`` of the parent. + + * ``postvisit(node, par_value, value)`` - should return ``value`` + ``par_value`` is as returned from ``previsit()`` of the parent, and ``value`` is as + returned from ``previsit()`` of this node itself. The return ``value`` is ignored except + the one for the root node, which is returned from the overall ``visit_tree()`` call. + + For the initial node, ``par_value`` is None. Either ``previsit`` and ``postvisit`` may be None. + """ + if not previsit: + previsit = lambda node, pvalue: (None, None) + if not postvisit: + postvisit = lambda node, pvalue, value: None + + iter_children = iter_children_func(node) + done = set() + ret = None + stack = [(node, None, _PREVISIT)] + while stack: + current, par_value, value = stack.pop() + if value is _PREVISIT: + assert current not in done # protect againt infinite loop in case of a bad tree. + done.add(current) + + pvalue, post_value = previsit(current, par_value) + stack.append((current, par_value, post_value)) + + # Insert all children in reverse order (so that first child ends up on top of the stack). + ins = len(stack) + for n in iter_children(current): + stack.insert(ins, (n, pvalue, _PREVISIT)) + else: + ret = postvisit(current, par_value, value) + return ret + + + +def walk(node): + """ + Recursively yield all descendant nodes in the tree starting at ``node`` (including ``node`` + itself), using depth-first pre-order traversal (yieling parents before their children). + + This is similar to ``ast.walk()``, but with a different order, and it works for both ``ast`` and + ``astroid`` trees. Also, as ``iter_children()``, it skips singleton nodes generated by ``ast``. + """ + iter_children = iter_children_func(node) + done = set() + stack = [node] + while stack: + current = stack.pop() + assert current not in done # protect againt infinite loop in case of a bad tree. + done.add(current) + + yield current + + # Insert all children in reverse order (so that first child ends up on top of the stack). + # This is faster than building a list and reversing it. + ins = len(stack) + for c in iter_children(current): + stack.insert(ins, c) + + +def replace(text, replacements): + """ + Replaces multiple slices of text with new values. This is a convenience method for making code + modifications of ranges e.g. as identified by ``ASTTokens.get_text_range(node)``. Replacements is + an iterable of ``(start, end, new_text)`` tuples. + + For example, ``replace("this is a test", [(0, 4, "X"), (8, 1, "THE")])`` produces + ``"X is THE test"``. + """ + p = 0 + parts = [] + for (start, end, new_text) in sorted(replacements): + parts.append(text[p:start]) + parts.append(new_text) + p = end + parts.append(text[p:]) + return ''.join(parts) + + +class NodeMethods(object): + """ + Helper to get `visit_{node_type}` methods given a node's class and cache the results. + """ + def __init__(self): + self._cache = {} + + def get(self, obj, cls): + """ + Using the lowercase name of the class as node_type, returns `obj.visit_{node_type}`, + or `obj.visit_default` if the type-specific method is not found. + """ + method = self._cache.get(cls) + if not method: + name = "visit_" + cls.__name__.lower() + method = getattr(obj, name, obj.visit_default) + self._cache[cls] = method + return method diff --git a/contrib/flake8/__init__.py b/contrib/flake8/__init__.py index 3bfa1a6..efb711c 100644 --- a/contrib/flake8/__init__.py +++ b/contrib/flake8/__init__.py @@ -1 +1,82 @@ -__version__ = '2.5.2' +"""Top-level module for Flake8. + +This module + +- initializes logging for the command-line tool +- tracks the version of the package +- provides a way to configure logging for the command-line tool + +.. autofunction:: flake8.configure_logging + +""" +import logging +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + """Shim for version of Python < 2.7.""" + + def emit(self, record): + """Do nothing.""" + pass +import sys + +LOG = logging.getLogger(__name__) +LOG.addHandler(NullHandler()) + +# Clean up after LOG config +del NullHandler + +__version__ = '3.5.0' +__version_info__ = tuple(int(i) for i in __version__.split('.') if i.isdigit()) + + +# There is nothing lower than logging.DEBUG (10) in the logging library, +# but we want an extra level to avoid being too verbose when using -vv. +_EXTRA_VERBOSE = 5 +logging.addLevelName(_EXTRA_VERBOSE, 'VERBOSE') + +_VERBOSITY_TO_LOG_LEVEL = { + # output more than warnings but not debugging info + 1: logging.INFO, # INFO is a numerical level of 20 + # output debugging information + 2: logging.DEBUG, # DEBUG is a numerical level of 10 + # output extra verbose debugging information + 3: _EXTRA_VERBOSE, +} + +LOG_FORMAT = ('%(name)-25s %(processName)-11s %(relativeCreated)6d ' + '%(levelname)-8s %(message)s') + + +def configure_logging(verbosity, filename=None, logformat=LOG_FORMAT): + """Configure logging for flake8. + + :param int verbosity: + How verbose to be in logging information. + :param str filename: + Name of the file to append log information to. + If ``None`` this will log to ``sys.stderr``. + If the name is "stdout" or "stderr" this will log to the appropriate + stream. + """ + if verbosity <= 0: + return + if verbosity > 3: + verbosity = 3 + + log_level = _VERBOSITY_TO_LOG_LEVEL[verbosity] + + if not filename or filename in ('stderr', 'stdout'): + fileobj = getattr(sys, filename or 'stderr') + handler_cls = logging.StreamHandler + else: + fileobj = filename + handler_cls = logging.FileHandler + + handler = handler_cls(fileobj) + handler.setFormatter(logging.Formatter(logformat)) + LOG.addHandler(handler) + LOG.setLevel(log_level) + LOG.debug('Added a %s logging handler to logger root at %s', + filename, __name__) diff --git a/contrib/flake8/__main__.py b/contrib/flake8/__main__.py index aaa497b..42bc428 100644 --- a/contrib/flake8/__main__.py +++ b/contrib/flake8/__main__.py @@ -1,4 +1,4 @@ -from flake8.main import main +"""Module allowing for ``python -m flake8 ...``.""" +from flake8.main import cli -# python -m flake8 (with Python >= 2.7) -main() +cli.main() diff --git a/contrib/flake8/_pyflakes.py b/contrib/flake8/_pyflakes.py deleted file mode 100644 index 976b2ab..0000000 --- a/contrib/flake8/_pyflakes.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- -try: - # The 'demandimport' breaks pyflakes and flake8._pyflakes - from mercurial import demandimport -except ImportError: - pass -else: - demandimport.disable() -import os - -import pep8 -import pyflakes -import pyflakes.checker - - -def patch_pyflakes(): - """Add error codes to Pyflakes messages.""" - codes = dict([line.split()[::-1] for line in ( - 'F401 UnusedImport', - 'F402 ImportShadowedByLoopVar', - 'F403 ImportStarUsed', - 'F404 LateFutureImport', - 'F810 Redefined', # XXX Obsolete? - 'F811 RedefinedWhileUnused', - 'F812 RedefinedInListComp', - 'F821 UndefinedName', - 'F822 UndefinedExport', - 'F823 UndefinedLocal', - 'F831 DuplicateArgument', - 'F841 UnusedVariable', - )]) - - for name, obj in vars(pyflakes.messages).items(): - if name[0].isupper() and obj.message: - obj.flake8_msg = '%s %s' % (codes.get(name, 'F999'), obj.message) -patch_pyflakes() - - -class FlakesChecker(pyflakes.checker.Checker): - """Subclass the Pyflakes checker to conform with the flake8 API.""" - name = 'pyflakes' - version = pyflakes.__version__ - - def __init__(self, tree, filename): - filename = pep8.normalize_paths(filename)[0] - withDoctest = self.withDoctest - included_by = [include for include in self.include_in_doctest - if include != '' and filename.startswith(include)] - if included_by: - withDoctest = True - - for exclude in self.exclude_from_doctest: - if exclude != '' and filename.startswith(exclude): - withDoctest = False - overlaped_by = [include for include in included_by - if include.startswith(exclude)] - - if overlaped_by: - withDoctest = True - - super(FlakesChecker, self).__init__(tree, filename, - withDoctest=withDoctest) - - @classmethod - def add_options(cls, parser): - parser.add_option('--builtins', - help="define more built-ins, comma separated") - parser.add_option('--doctests', default=False, action='store_true', - help="check syntax of the doctests") - parser.add_option('--include-in-doctest', default='', - dest='include_in_doctest', - help='Run doctests only on these files', - type='string') - parser.add_option('--exclude-from-doctest', default='', - dest='exclude_from_doctest', - help='Skip these files when running doctests', - type='string') - parser.config_options.extend(['builtins', 'doctests', - 'include-in-doctest', - 'exclude-from-doctest']) - - @classmethod - def parse_options(cls, options): - if options.builtins: - cls.builtIns = cls.builtIns.union(options.builtins.split(',')) - cls.withDoctest = options.doctests - - included_files = [] - for included_file in options.include_in_doctest.split(','): - if included_file == '': - continue - if not included_file.startswith((os.sep, './', '~/')): - included_files.append('./' + included_file) - else: - included_files.append(included_file) - cls.include_in_doctest = pep8.normalize_paths(','.join(included_files)) - - excluded_files = [] - for excluded_file in options.exclude_from_doctest.split(','): - if excluded_file == '': - continue - if not excluded_file.startswith((os.sep, './', '~/')): - excluded_files.append('./' + excluded_file) - else: - excluded_files.append(excluded_file) - cls.exclude_from_doctest = pep8.normalize_paths( - ','.join(excluded_files)) - - inc_exc = set(cls.include_in_doctest).intersection( - set(cls.exclude_from_doctest)) - if inc_exc: - raise ValueError('"%s" was specified in both the ' - 'include-in-doctest and exclude-from-doctest ' - 'options. You are not allowed to specify it in ' - 'both for doctesting.' % inc_exc) - - def run(self): - for m in self.messages: - col = getattr(m, 'col', 0) - yield m.lineno, col, (m.flake8_msg % m.message_args), m.__class__ diff --git a/contrib/flake8/api/__init__.py b/contrib/flake8/api/__init__.py new file mode 100644 index 0000000..c2eefbe --- /dev/null +++ b/contrib/flake8/api/__init__.py @@ -0,0 +1,5 @@ +"""Module containing all public entry-points for Flake8. + +This is the only submodule in Flake8 with a guaranteed stable API. All other +submodules are considered internal only and are subject to change. +""" diff --git a/contrib/flake8/api/legacy.py b/contrib/flake8/api/legacy.py new file mode 100644 index 0000000..b332860 --- /dev/null +++ b/contrib/flake8/api/legacy.py @@ -0,0 +1,202 @@ +"""Module containing shims around Flake8 2.x behaviour. + +Previously, users would import :func:`get_style_guide` from ``flake8.engine``. +In 3.0 we no longer have an "engine" module but we maintain the API from it. +""" +import logging +import os.path + +import flake8 +from flake8.formatting import base as formatter +from flake8.main import application as app + +LOG = logging.getLogger(__name__) + + +__all__ = ('get_style_guide',) + + +def get_style_guide(**kwargs): + r"""Provision a StyleGuide for use. + + :param \*\*kwargs: + Keyword arguments that provide some options for the StyleGuide. + :returns: + An initialized StyleGuide + :rtype: + :class:`StyleGuide` + """ + application = app.Application() + application.parse_preliminary_options_and_args([]) + flake8.configure_logging( + application.prelim_opts.verbose, application.prelim_opts.output_file) + application.make_config_finder() + application.find_plugins() + application.register_plugin_options() + application.parse_configuration_and_cli([]) + # We basically want application.initialize to be called but with these + # options set instead before we make our formatter, notifier, internal + # style guide and file checker manager. + options = application.options + for key, value in kwargs.items(): + try: + getattr(options, key) + setattr(options, key, value) + except AttributeError: + LOG.error('Could not update option "%s"', key) + application.make_formatter() + application.make_notifier() + application.make_guide() + application.make_file_checker_manager() + return StyleGuide(application) + + +class StyleGuide(object): + """Public facing object that mimic's Flake8 2.0's StyleGuide. + + .. note:: + + There are important changes in how this object behaves compared to + the StyleGuide object provided in Flake8 2.x. + + .. warning:: + + This object should not be instantiated directly by users. + + .. versionchanged:: 3.0.0 + """ + + def __init__(self, application): + """Initialize our StyleGuide.""" + self._application = application + self._file_checker_manager = application.file_checker_manager + + @property + def options(self): + """Return application's options. + + An instance of :class:`optparse.Values` containing parsed options. + """ + return self._application.options + + @property + def paths(self): + """Return the extra arguments passed as paths.""" + return self._application.paths + + def check_files(self, paths=None): + """Run collected checks on the files provided. + + This will check the files passed in and return a :class:`Report` + instance. + + :param list paths: + List of filenames (or paths) to check. + :returns: + Object that mimic's Flake8 2.0's Reporter class. + :rtype: + flake8.api.legacy.Report + """ + self._application.run_checks(paths) + self._application.report_errors() + return Report(self._application) + + def excluded(self, filename, parent=None): + """Determine if a file is excluded. + + :param str filename: + Path to the file to check if it is excluded. + :param str parent: + Name of the parent directory containing the file. + :returns: + True if the filename is excluded, False otherwise. + :rtype: + bool + """ + return (self._file_checker_manager.is_path_excluded(filename) or + (parent and + self._file_checker_manager.is_path_excluded( + os.path.join(parent, filename)))) + + def init_report(self, reporter=None): + """Set up a formatter for this run of Flake8.""" + if reporter is None: + return + if not issubclass(reporter, formatter.BaseFormatter): + raise ValueError("Report should be subclass of " + "flake8.formatter.BaseFormatter.") + self._application.formatter = None + self._application.make_formatter(reporter) + self._application.guide = None + # NOTE(sigmavirus24): This isn't the intended use of + # Application#make_guide but it works pretty well. + # Stop cringing... I know it's gross. + self._application.make_guide() + self._application.file_checker_manager = None + self._application.make_file_checker_manager() + + def input_file(self, filename, lines=None, expected=None, line_offset=0): + """Run collected checks on a single file. + + This will check the file passed in and return a :class:`Report` + instance. + + :param str filename: + The path to the file to check. + :param list lines: + Ignored since Flake8 3.0. + :param expected: + Ignored since Flake8 3.0. + :param int line_offset: + Ignored since Flake8 3.0. + :returns: + Object that mimic's Flake8 2.0's Reporter class. + :rtype: + flake8.api.legacy.Report + """ + return self.check_files([filename]) + + +class Report(object): + """Public facing object that mimic's Flake8 2.0's API. + + .. note:: + + There are important changes in how this object behaves compared to + the object provided in Flake8 2.x. + + .. warning:: + + This should not be instantiated by users. + + .. versionchanged:: 3.0.0 + """ + + def __init__(self, application): + """Initialize the Report for the user. + + .. warning:: This should not be instantiated by users. + """ + self._application = application + self._style_guide = application.guide + self._stats = self._style_guide.stats + + @property + def total_errors(self): + """Return the total number of errors.""" + return self._application.result_count + + def get_statistics(self, violation): + """Get the list of occurrences of a violation. + + :returns: + List of occurrences of a violation formatted as: + {Count} {Error Code} {Message}, e.g., + ``8 E531 Some error message about the error`` + :rtype: + list + """ + return [ + '{} {} {}'.format(s.count, s.error_code, s.message) + for s in self._stats.statistics_for(violation) + ] diff --git a/contrib/flake8/callbacks.py b/contrib/flake8/callbacks.py deleted file mode 100644 index 3767f30..0000000 --- a/contrib/flake8/callbacks.py +++ /dev/null @@ -1,27 +0,0 @@ -import atexit -import sys - - -def install_vcs_hook(option, option_str, value, parser): - # For now, there's no way to affect a change in how pep8 processes - # options. If no args are provided and there's no config file present, - # it will error out because no input was provided. To get around this, - # when we're using --install-hook, we'll say that there were arguments so - # we can actually attempt to install the hook. - # See: https://gitlab.com/pycqa/flake8/issues/2 and - # https://github.com/jcrocholl/pep8/blob/4c5bf00cb613be617c7f48d3b2b82a1c7b895ac1/pep8.py#L1912 - # for more context. - parser.values.install_hook = True - parser.rargs.append('.') - - -def restore_stdout(old_stdout): - sys.stdout.close() - sys.stdout = old_stdout - - -def redirect_stdout(option, option_str, value, parser): - fd = open(value, 'w') - old_stdout, sys.stdout = sys.stdout, fd - - atexit.register(restore_stdout, old_stdout) diff --git a/contrib/flake8/checker.py b/contrib/flake8/checker.py new file mode 100644 index 0000000..6e53cb5 --- /dev/null +++ b/contrib/flake8/checker.py @@ -0,0 +1,659 @@ +"""Checker Manager and Checker classes.""" +import collections +import errno +import logging +import os +import signal +import sys +import tokenize + +try: + import multiprocessing +except ImportError: + multiprocessing = None + +from flake8 import defaults +from flake8 import exceptions +from flake8 import processor +from flake8 import utils + +LOG = logging.getLogger(__name__) + +SERIAL_RETRY_ERRNOS = { + # ENOSPC: Added by sigmavirus24 + # > On some operating systems (OSX), multiprocessing may cause an + # > ENOSPC error while trying to trying to create a Semaphore. + # > In those cases, we should replace the customized Queue Report + # > class with pep8's StandardReport class to ensure users don't run + # > into this problem. + # > (See also: https://gitlab.com/pycqa/flake8/issues/74) + errno.ENOSPC, + # NOTE(sigmavirus24): When adding to this list, include the reasoning + # on the lines before the error code and always append your error + # code. Further, please always add a trailing `,` to reduce the visual + # noise in diffs. +} + + +class Manager(object): + """Manage the parallelism and checker instances for each plugin and file. + + This class will be responsible for the following: + + - Determining the parallelism of Flake8, e.g.: + + * Do we use :mod:`multiprocessing` or is it unavailable? + + * Do we automatically decide on the number of jobs to use or did the + user provide that? + + - Falling back to a serial way of processing files if we run into an + OSError related to :mod:`multiprocessing` + + - Organizing the results of each checker so we can group the output + together and make our output deterministic. + """ + + def __init__(self, style_guide, arguments, checker_plugins): + """Initialize our Manager instance. + + :param style_guide: + The instantiated style guide for this instance of Flake8. + :type style_guide: + flake8.style_guide.StyleGuide + :param list arguments: + The extra arguments parsed from the CLI (if any) + :param checker_plugins: + The plugins representing checks parsed from entry-points. + :type checker_plugins: + flake8.plugins.manager.Checkers + """ + self.arguments = arguments + self.style_guide = style_guide + self.options = style_guide.options + self.checks = checker_plugins + self.jobs = self._job_count() + self.using_multiprocessing = self.jobs > 1 + self.pool = None + self.processes = [] + self.checkers = [] + self.statistics = { + 'files': 0, + 'logical lines': 0, + 'physical lines': 0, + 'tokens': 0, + } + + if self.using_multiprocessing: + try: + self.pool = multiprocessing.Pool(self.jobs, _pool_init) + except OSError as oserr: + if oserr.errno not in SERIAL_RETRY_ERRNOS: + raise + self.using_multiprocessing = False + + def _process_statistics(self): + for checker in self.checkers: + for statistic in defaults.STATISTIC_NAMES: + self.statistics[statistic] += checker.statistics[statistic] + self.statistics['files'] += len(self.checkers) + + def _job_count(self): + # type: () -> int + # First we walk through all of our error cases: + # - multiprocessing library is not present + # - we're running on windows in which case we know we have significant + # implemenation issues + # - the user provided stdin and that's not something we can handle + # well + # - we're processing a diff, which again does not work well with + # multiprocessing and which really shouldn't require multiprocessing + # - the user provided some awful input + if not multiprocessing: + LOG.warning('The multiprocessing module is not available. ' + 'Ignoring --jobs arguments.') + return 0 + + if (utils.is_windows() and + not utils.can_run_multiprocessing_on_windows()): + LOG.warning('The --jobs option is not available on Windows due to' + ' a bug (https://bugs.python.org/issue27649) in ' + 'Python 2.7.11+ and 3.3+. We have detected that you ' + 'are running an unsupported version of Python on ' + 'Windows. Ignoring --jobs arguments.') + return 0 + + if utils.is_using_stdin(self.arguments): + LOG.warning('The --jobs option is not compatible with supplying ' + 'input using - . Ignoring --jobs arguments.') + return 0 + + if self.options.diff: + LOG.warning('The --diff option was specified with --jobs but ' + 'they are not compatible. Ignoring --jobs arguments.') + return 0 + + jobs = self.options.jobs + if jobs != 'auto' and not jobs.isdigit(): + LOG.warning('"%s" is not a valid parameter to --jobs. Must be one ' + 'of "auto" or a numerical value, e.g., 4.', jobs) + return 0 + + # If the value is "auto", we want to let the multiprocessing library + # decide the number based on the number of CPUs. However, if that + # function is not implemented for this particular value of Python we + # default to 1 + if jobs == 'auto': + try: + return multiprocessing.cpu_count() + except NotImplementedError: + return 0 + + # Otherwise, we know jobs should be an integer and we can just convert + # it to an integer + return int(jobs) + + def _handle_results(self, filename, results): + style_guide = self.style_guide + reported_results_count = 0 + for (error_code, line_number, column, text, physical_line) in results: + reported_results_count += style_guide.handle_error( + code=error_code, + filename=filename, + line_number=line_number, + column_number=column, + text=text, + physical_line=physical_line, + ) + return reported_results_count + + def is_path_excluded(self, path): + # type: (str) -> bool + """Check if a path is excluded. + + :param str path: + Path to check against the exclude patterns. + :returns: + True if there are exclude patterns and the path matches, + otherwise False. + :rtype: + bool + """ + if path == '-': + if self.options.stdin_display_name == 'stdin': + return False + path = self.options.stdin_display_name + + exclude = self.options.exclude + if not exclude: + return False + basename = os.path.basename(path) + if utils.fnmatch(basename, exclude): + LOG.debug('"%s" has been excluded', basename) + return True + + absolute_path = os.path.abspath(path) + match = utils.fnmatch(absolute_path, exclude) + LOG.debug('"%s" has %sbeen excluded', absolute_path, + '' if match else 'not ') + return match + + def make_checkers(self, paths=None): + # type: (List[str]) -> NoneType + """Create checkers for each file.""" + if paths is None: + paths = self.arguments + + if not paths: + paths = ['.'] + + filename_patterns = self.options.filename + running_from_vcs = self.options._running_from_vcs + running_from_diff = self.options.diff + + # NOTE(sigmavirus24): Yes this is a little unsightly, but it's our + # best solution right now. + def should_create_file_checker(filename, argument): + """Determine if we should create a file checker.""" + matches_filename_patterns = utils.fnmatch( + filename, filename_patterns + ) + is_stdin = filename == '-' + file_exists = os.path.exists(filename) + # NOTE(sigmavirus24): If a user explicitly specifies something, + # e.g, ``flake8 bin/script`` then we should run Flake8 against + # that. Since should_create_file_checker looks to see if the + # filename patterns match the filename, we want to skip that in + # the event that the argument and the filename are identical. + # If it was specified explicitly, the user intended for it to be + # checked. + explicitly_provided = (not running_from_vcs and + not running_from_diff and + (argument == filename)) + return ((file_exists and + (explicitly_provided or matches_filename_patterns)) or + is_stdin) + + checks = self.checks.to_dictionary() + checkers = ( + FileChecker(filename, checks, self.options) + for argument in paths + for filename in utils.filenames_from(argument, + self.is_path_excluded) + if should_create_file_checker(filename, argument) + ) + self.checkers = [ + checker for checker in checkers if checker.should_process + ] + LOG.info('Checking %d files', len(self.checkers)) + + def report(self): + # type: () -> (int, int) + """Report all of the errors found in the managed file checkers. + + This iterates over each of the checkers and reports the errors sorted + by line number. + + :returns: + A tuple of the total results found and the results reported. + :rtype: + tuple(int, int) + """ + results_reported = results_found = 0 + for checker in self.checkers: + results = sorted(checker.results, key=lambda tup: (tup[1], tup[2])) + filename = checker.display_name + with self.style_guide.processing_file(filename): + results_reported += self._handle_results(filename, results) + results_found += len(results) + return (results_found, results_reported) + + def _force_cleanup(self): + if self.pool is not None: + self.pool.terminate() + self.pool.join() + + def run_parallel(self): + """Run the checkers in parallel.""" + final_results = collections.defaultdict(list) + final_statistics = collections.defaultdict(dict) + pool_map = self.pool.imap_unordered( + _run_checks, + self.checkers, + chunksize=calculate_pool_chunksize( + len(self.checkers), + self.jobs, + ), + ) + for ret in pool_map: + filename, results, statistics = ret + final_results[filename] = results + final_statistics[filename] = statistics + self.pool.close() + self.pool.join() + self.pool = None + + for checker in self.checkers: + filename = checker.display_name + checker.results = sorted(final_results[filename], + key=lambda tup: (tup[2], tup[2])) + checker.statistics = final_statistics[filename] + + def run_serial(self): + """Run the checkers in serial.""" + for checker in self.checkers: + checker.run_checks() + + def run(self): + """Run all the checkers. + + This will intelligently decide whether to run the checks in parallel + or whether to run them in serial. + + If running the checks in parallel causes a problem (e.g., + https://gitlab.com/pycqa/flake8/issues/74) this also implements + fallback to serial processing. + """ + try: + if self.using_multiprocessing: + self.run_parallel() + else: + self.run_serial() + except OSError as oserr: + if oserr.errno not in SERIAL_RETRY_ERRNOS: + LOG.exception(oserr) + raise + LOG.warning('Running in serial after OS exception, %r', oserr) + self.run_serial() + except KeyboardInterrupt: + LOG.warning('Flake8 was interrupted by the user') + raise exceptions.EarlyQuit('Early quit while running checks') + finally: + self._force_cleanup() + + def start(self, paths=None): + """Start checking files. + + :param list paths: + Path names to check. This is passed directly to + :meth:`~Manager.make_checkers`. + """ + LOG.info('Making checkers') + self.make_checkers(paths) + + def stop(self): + """Stop checking files.""" + self._process_statistics() + for proc in self.processes: + LOG.info('Joining %s to the main process', proc.name) + proc.join() + + +class FileChecker(object): + """Manage running checks for a file and aggregate the results.""" + + def __init__(self, filename, checks, options): + """Initialize our file checker. + + :param str filename: + Name of the file to check. + :param checks: + The plugins registered to check the file. + :type checks: + dict + :param options: + Parsed option values from config and command-line. + :type options: + optparse.Values + """ + self.options = options + self.filename = filename + self.checks = checks + self.results = [] + self.statistics = { + 'tokens': 0, + 'logical lines': 0, + 'physical lines': 0, + } + self.processor = self._make_processor() + self.display_name = filename + self.should_process = False + if self.processor is not None: + self.display_name = self.processor.filename + self.should_process = not self.processor.should_ignore_file() + self.statistics['physical lines'] = len(self.processor.lines) + + def __repr__(self): + """Provide helpful debugging representation.""" + return 'FileChecker for {}'.format(self.filename) + + def _make_processor(self): + try: + return processor.FileProcessor(self.filename, self.options) + except IOError: + # If we can not read the file due to an IOError (e.g., the file + # does not exist or we do not have the permissions to open it) + # then we need to format that exception for the user. + # NOTE(sigmavirus24): Historically, pep8 has always reported this + # as an E902. We probably *want* a better error code for this + # going forward. + (exc_type, exception) = sys.exc_info()[:2] + message = '{0}: {1}'.format(exc_type.__name__, exception) + self.report('E902', 0, 0, message) + return None + + def report(self, error_code, line_number, column, text, line=None): + # type: (str, int, int, str) -> str + """Report an error by storing it in the results list.""" + if error_code is None: + error_code, text = text.split(' ', 1) + + physical_line = line + # If we're recovering from a problem in _make_processor, we will not + # have this attribute. + if not physical_line and getattr(self, 'processor', None): + physical_line = self.processor.line_for(line_number) + + error = (error_code, line_number, column, text, physical_line) + self.results.append(error) + return error_code + + def run_check(self, plugin, **arguments): + """Run the check in a single plugin.""" + LOG.debug('Running %r with %r', plugin, arguments) + try: + self.processor.keyword_arguments_for( + plugin['parameters'], + arguments, + ) + except AttributeError as ae: + LOG.error('Plugin requested unknown parameters.') + raise exceptions.PluginRequestedUnknownParameters( + plugin=plugin, + exception=ae, + ) + return plugin['plugin'](**arguments) + + @staticmethod + def _extract_syntax_information(exception): + token = () + if len(exception.args) > 1: + token = exception.args[1] + if len(token) > 2: + row, column = token[1:3] + else: + row, column = (1, 0) + + if column > 0 and token and isinstance(exception, SyntaxError): + # NOTE(sigmavirus24): SyntaxErrors report 1-indexed column + # numbers. We need to decrement the column number by 1 at + # least. + column_offset = 1 + row_offset = 0 + # See also: https://gitlab.com/pycqa/flake8/issues/237 + physical_line = token[-1] + + # NOTE(sigmavirus24): Not all "tokens" have a string as the last + # argument. In this event, let's skip trying to find the correct + # column and row values. + if physical_line is not None: + # NOTE(sigmavirus24): SyntaxErrors also don't exactly have a + # "physical" line so much as what was accumulated by the point + # tokenizing failed. + # See also: https://gitlab.com/pycqa/flake8/issues/237 + lines = physical_line.rstrip('\n').split('\n') + row_offset = len(lines) - 1 + logical_line = lines[0] + logical_line_length = len(logical_line) + if column > logical_line_length: + column = logical_line_length + row -= row_offset + column -= column_offset + return row, column + + def run_ast_checks(self): + """Run all checks expecting an abstract syntax tree.""" + try: + ast = self.processor.build_ast() + except (ValueError, SyntaxError, TypeError): + (exc_type, exception) = sys.exc_info()[:2] + row, column = self._extract_syntax_information(exception) + self.report('E999', row, column, '%s: %s' % + (exc_type.__name__, exception.args[0])) + return + + for plugin in self.checks['ast_plugins']: + checker = self.run_check(plugin, tree=ast) + # If the plugin uses a class, call the run method of it, otherwise + # the call should return something iterable itself + try: + runner = checker.run() + except AttributeError: + runner = checker + for (line_number, offset, text, check) in runner: + self.report( + error_code=None, + line_number=line_number, + column=offset, + text=text, + ) + + def run_logical_checks(self): + """Run all checks expecting a logical line.""" + comments, logical_line, mapping = self.processor.build_logical_line() + if not mapping: + return + self.processor.update_state(mapping) + + LOG.debug('Logical line: "%s"', logical_line.rstrip()) + + for plugin in self.checks['logical_line_plugins']: + self.processor.update_checker_state_for(plugin) + results = self.run_check(plugin, logical_line=logical_line) or () + for offset, text in results: + offset = find_offset(offset, mapping) + line_number, column_offset = offset + self.report( + error_code=None, + line_number=line_number, + column=column_offset, + text=text, + ) + + self.processor.next_logical_line() + + def run_physical_checks(self, physical_line, override_error_line=None): + """Run all checks for a given physical line.""" + for plugin in self.checks['physical_line_plugins']: + self.processor.update_checker_state_for(plugin) + result = self.run_check(plugin, physical_line=physical_line) + if result is not None: + column_offset, text = result + error_code = self.report( + error_code=None, + line_number=self.processor.line_number, + column=column_offset, + text=text, + line=(override_error_line or physical_line), + ) + + self.processor.check_physical_error(error_code, physical_line) + + def process_tokens(self): + """Process tokens and trigger checks. + + This can raise a :class:`flake8.exceptions.InvalidSyntax` exception. + Instead of using this directly, you should use + :meth:`flake8.checker.FileChecker.run_checks`. + """ + parens = 0 + statistics = self.statistics + file_processor = self.processor + for token in file_processor.generate_tokens(): + statistics['tokens'] += 1 + self.check_physical_eol(token) + token_type, text = token[0:2] + processor.log_token(LOG, token) + if token_type == tokenize.OP: + parens = processor.count_parentheses(parens, text) + elif parens == 0: + if processor.token_is_newline(token): + self.handle_newline(token_type) + elif (processor.token_is_comment(token) and + len(file_processor.tokens) == 1): + self.handle_comment(token, text) + + if file_processor.tokens: + # If any tokens are left over, process them + self.run_physical_checks(file_processor.lines[-1]) + self.run_logical_checks() + + def run_checks(self): + """Run checks against the file.""" + try: + self.process_tokens() + except exceptions.InvalidSyntax as exc: + self.report(exc.error_code, exc.line_number, exc.column_number, + exc.error_message) + + self.run_ast_checks() + + logical_lines = self.processor.statistics['logical lines'] + self.statistics['logical lines'] = logical_lines + return self.filename, self.results, self.statistics + + def handle_comment(self, token, token_text): + """Handle the logic when encountering a comment token.""" + # The comment also ends a physical line + token = list(token) + token[1] = token_text.rstrip('\r\n') + token[3] = (token[2][0], token[2][1] + len(token[1])) + self.processor.tokens = [tuple(token)] + self.run_logical_checks() + + def handle_newline(self, token_type): + """Handle the logic when encountering a newline token.""" + if token_type == tokenize.NEWLINE: + self.run_logical_checks() + self.processor.reset_blank_before() + elif len(self.processor.tokens) == 1: + # The physical line contains only this token. + self.processor.visited_new_blank_line() + self.processor.delete_first_token() + else: + self.run_logical_checks() + + def check_physical_eol(self, token): + """Run physical checks if and only if it is at the end of the line.""" + if processor.is_eol_token(token): + # Obviously, a newline token ends a single physical line. + self.run_physical_checks(token[4]) + elif processor.is_multiline_string(token): + # Less obviously, a string that contains newlines is a + # multiline string, either triple-quoted or with internal + # newlines backslash-escaped. Check every physical line in the + # string *except* for the last one: its newline is outside of + # the multiline string, so we consider it a regular physical + # line, and will check it like any other physical line. + # + # Subtleties: + # - have to wind self.line_number back because initially it + # points to the last line of the string, and we want + # check_physical() to give accurate feedback + line_no = token[2][0] + with self.processor.inside_multiline(line_number=line_no): + for line in self.processor.split_line(token): + self.run_physical_checks(line + '\n', + override_error_line=token[4]) + + +def _pool_init(): + """Ensure correct signaling of ^C using multiprocessing.Pool.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + + +def calculate_pool_chunksize(num_checkers, num_jobs): + """Determine the chunksize for the multiprocessing Pool. + + - For chunksize, see: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.imap # noqa + - This formula, while not perfect, aims to give each worker two batches of + work. + - See: https://gitlab.com/pycqa/flake8/merge_requests/156#note_18878876 + - See: https://gitlab.com/pycqa/flake8/issues/265 + """ + return max(num_checkers // (num_jobs * 2), 1) + + +def _run_checks(checker): + return checker.run_checks() + + +def find_offset(offset, mapping): + """Find the offset tuple for a single offset.""" + if isinstance(offset, tuple): + return offset + + for token_offset, position in mapping: + if offset <= token_offset: + break + return (position[0], position[1] + offset - token_offset) diff --git a/contrib/flake8/compat.py b/contrib/flake8/compat.py deleted file mode 100644 index 9bd00a7..0000000 --- a/contrib/flake8/compat.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -"""Compatibility shims for Flake8.""" -import os.path -import sys - - -def relpath(path, start='.'): - """Wallpaper over the differences between 2.6 and newer versions.""" - if sys.version_info < (2, 7) and path.startswith(start): - return path[len(start):] - else: - return os.path.relpath(path, start=start) diff --git a/contrib/flake8/defaults.py b/contrib/flake8/defaults.py new file mode 100644 index 0000000..55cb48a --- /dev/null +++ b/contrib/flake8/defaults.py @@ -0,0 +1,53 @@ +"""Constants that define defaults.""" +import re + +EXCLUDE = ( + '.svn', + 'CVS', + '.bzr', + '.hg', + '.git', + '__pycache__', + '.tox', + '.eggs', + '*.egg', +) +IGNORE = ( + 'E121', + 'E123', + 'E126', + 'E226', + 'E24', + 'E704', + 'W503', + 'W504', +) +SELECT = ('E', 'F', 'W', 'C90') +MAX_LINE_LENGTH = 79 + +TRUTHY_VALUES = {'true', '1', 't'} + +# Other constants +WHITESPACE = frozenset(' \t') + +STATISTIC_NAMES = ( + 'logical lines', + 'physical lines', + 'tokens', +) + +NOQA_INLINE_REGEXP = re.compile( + # We're looking for items that look like this: + # ``# noqa`` + # ``# noqa: E123`` + # ``# noqa: E123,W451,F921`` + # ``# NoQA: E123,W451,F921`` + # ``# NOQA: E123,W451,F921`` + # We do not care about the ``: `` that follows ``noqa`` + # We do not care about the casing of ``noqa`` + # We want a comma-separated list of errors + '# noqa(?:: (?P([A-Z][0-9]+(?:[,\s]+)?)+))?', + re.IGNORECASE +) + +NOQA_FILE = re.compile(r'\s*# flake8[:=]\s*noqa', re.I) diff --git a/contrib/flake8/engine.py b/contrib/flake8/engine.py deleted file mode 100644 index 5ee91a5..0000000 --- a/contrib/flake8/engine.py +++ /dev/null @@ -1,316 +0,0 @@ -# -*- coding: utf-8 -*- -import errno -import io -import platform -import re -import sys -import warnings - -import pep8 - -from flake8 import __version__ -from flake8 import callbacks -from flake8.reporter import (multiprocessing, BaseQReport, FileQReport, - QueueReport) -from flake8 import util - -_flake8_noqa = re.compile(r'\s*# flake8[:=]\s*noqa', re.I).search - -EXTRA_EXCLUDE = ['.tox', '.eggs', '*.egg'] - -pep8.PROJECT_CONFIG += ('.flake8',) - - -def _load_entry_point(entry_point, verify_requirements): - """Based on the version of setuptools load an entry-point correctly. - - setuptools 11.3 deprecated `require=False` in the call to EntryPoint.load. - To load entry points correctly after that without requiring all - dependencies be present, the proper way is to call EntryPoint.resolve. - - This function will provide backwards compatibility for older versions of - setuptools while also ensuring we do the right thing for the future. - """ - if hasattr(entry_point, 'resolve') and hasattr(entry_point, 'require'): - if verify_requirements: - entry_point.require() - plugin = entry_point.resolve() - else: - plugin = entry_point.load(require=verify_requirements) - - return plugin - - -def _register_extensions(): - """Register all the extensions.""" - extensions = util.OrderedSet() - extensions.add(('pep8', pep8.__version__)) - parser_hooks = [] - options_hooks = [] - ignored_hooks = [] - try: - from pkg_resources import iter_entry_points - except ImportError: - pass - else: - for entry in iter_entry_points('flake8.extension'): - # Do not verify that the requirements versions are valid - checker = _load_entry_point(entry, verify_requirements=False) - pep8.register_check(checker, codes=[entry.name]) - extensions.add((checker.name, checker.version)) - if hasattr(checker, 'add_options'): - parser_hooks.append(checker.add_options) - if hasattr(checker, 'parse_options'): - options_hooks.append(checker.parse_options) - if getattr(checker, 'off_by_default', False) is True: - ignored_hooks.append(entry.name) - return extensions, parser_hooks, options_hooks, ignored_hooks - - -def get_parser(): - """This returns an instance of optparse.OptionParser with all the - extensions registered and options set. This wraps ``pep8.get_parser``. - """ - (extensions, parser_hooks, options_hooks, ignored) = _register_extensions() - details = ', '.join('%s: %s' % ext for ext in extensions) - python_version = get_python_version() - parser = pep8.get_parser('flake8', '%s (%s) %s' % ( - __version__, details, python_version - )) - for opt in ('--repeat', '--testsuite', '--doctest'): - try: - parser.remove_option(opt) - except ValueError: - pass - - if multiprocessing: - parser.config_options.append('jobs') - parser.add_option('-j', '--jobs', type='string', default='auto', - help="number of jobs to run simultaneously, " - "or 'auto'. This is ignored on Windows.") - - parser.add_option('--exit-zero', action='store_true', - help="exit with code 0 even if there are errors") - for parser_hook in parser_hooks: - parser_hook(parser) - # See comment above regarding why this has to be a callback. - parser.add_option('--install-hook', default=False, dest='install_hook', - help='Install the appropriate hook for this ' - 'repository.', action='callback', - callback=callbacks.install_vcs_hook) - parser.add_option('--output-file', default=None, - help='Redirect report to a file.', - type='string', nargs=1, action='callback', - callback=callbacks.redirect_stdout) - parser.add_option('--enable-extensions', default='', - dest='enabled_extensions', - help='Enable plugins and extensions that are disabled ' - 'by default', - type='string') - parser.config_options.extend(['output_file', 'enable_extensions']) - parser.ignored_extensions = ignored - return parser, options_hooks - - -class NoQAStyleGuide(pep8.StyleGuide): - - def input_file(self, filename, lines=None, expected=None, line_offset=0): - """Run all checks on a Python source file.""" - if self.options.verbose: - print('checking %s' % filename) - fchecker = self.checker_class( - filename, lines=lines, options=self.options) - # Any "flake8: noqa" comments to ignore the entire file? - if any(_flake8_noqa(line) for line in fchecker.lines): - return 0 - return fchecker.check_all(expected=expected, line_offset=line_offset) - - -class StyleGuide(object): - """A wrapper StyleGuide object for Flake8 usage. - - This allows for OSErrors to be caught in the styleguide and special logic - to be used to handle those errors. - """ - - # Reasoning for error numbers is in-line below - serial_retry_errors = set([ - # ENOSPC: Added by sigmavirus24 - # > On some operating systems (OSX), multiprocessing may cause an - # > ENOSPC error while trying to trying to create a Semaphore. - # > In those cases, we should replace the customized Queue Report - # > class with pep8's StandardReport class to ensure users don't run - # > into this problem. - # > (See also: https://gitlab.com/pycqa/flake8/issues/74) - errno.ENOSPC, - # NOTE(sigmavirus24): When adding to this list, include the reasoning - # on the lines before the error code and always append your error - # code. Further, please always add a trailing `,` to reduce the visual - # noise in diffs. - ]) - - def __init__(self, **kwargs): - # This allows us to inject a mocked StyleGuide in the tests. - self._styleguide = kwargs.pop('styleguide', NoQAStyleGuide(**kwargs)) - - @property - def options(self): - return self._styleguide.options - - @property - def paths(self): - return self._styleguide.paths - - def _retry_serial(self, func, *args, **kwargs): - """This will retry the passed function in serial if necessary. - - In the event that we encounter an OSError with an errno in - :attr:`serial_retry_errors`, this function will retry this function - using pep8's default Report class which operates in serial. - """ - try: - return func(*args, **kwargs) - except OSError as oserr: - if oserr.errno in self.serial_retry_errors: - self.init_report(pep8.StandardReport) - else: - raise - return func(*args, **kwargs) - - def check_files(self, paths=None): - return self._retry_serial(self._styleguide.check_files, paths=paths) - - def excluded(self, filename, parent=None): - return self._styleguide.excluded(filename, parent=parent) - - def init_report(self, reporter=None): - return self._styleguide.init_report(reporter) - - def input_file(self, filename, lines=None, expected=None, line_offset=0): - return self._retry_serial( - self._styleguide.input_file, - filename=filename, - lines=lines, - expected=expected, - line_offset=line_offset, - ) - - -def _parse_multi_options(options, split_token=','): - r"""Split and strip and discard empties. - - Turns the following: - - A, - B, - - into ["A", "B"]. - - Credit: Kristian Glass as contributed to pep8 - """ - if options: - return [o.strip() for o in options.split(split_token) if o.strip()] - else: - return options - - -def _disable_extensions(parser, options): - ignored_extensions = set(getattr(parser, 'ignored_extensions', [])) - enabled = set(_parse_multi_options(options.enabled_extensions)) - - # Remove any of the selected extensions from the extensions ignored by - # default. - ignored_extensions -= enabled - - # Whatever is left afterwards should be unioned with options.ignore and - # options.ignore should be updated with that. - options.ignore = tuple(ignored_extensions.union(options.ignore)) - - -def get_style_guide(**kwargs): - """Parse the options and configure the checker. This returns a sub-class - of ``pep8.StyleGuide``.""" - kwargs['parser'], options_hooks = get_parser() - styleguide = StyleGuide(**kwargs) - options = styleguide.options - _disable_extensions(kwargs['parser'], options) - - if options.exclude and not isinstance(options.exclude, list): - options.exclude = pep8.normalize_paths(options.exclude) - elif not options.exclude: - options.exclude = [] - - # Add patterns in EXTRA_EXCLUDE to the list of excluded patterns - options.exclude.extend(pep8.normalize_paths(EXTRA_EXCLUDE)) - - for options_hook in options_hooks: - options_hook(options) - - if util.warn_when_using_jobs(options): - if not multiprocessing: - warnings.warn("The multiprocessing module is not available. " - "Ignoring --jobs arguments.") - if util.is_windows(): - warnings.warn("The --jobs option is not available on Windows. " - "Ignoring --jobs arguments.") - if util.is_using_stdin(styleguide.paths): - warnings.warn("The --jobs option is not compatible with supplying " - "input using - . Ignoring --jobs arguments.") - if options.diff: - warnings.warn("The --diff option was specified with --jobs but " - "they are not compatible. Ignoring --jobs arguments." - ) - - if options.diff: - options.jobs = None - - force_disable_jobs = util.force_disable_jobs(styleguide) - - if multiprocessing and options.jobs and not force_disable_jobs: - if options.jobs.isdigit(): - n_jobs = int(options.jobs) - else: - try: - n_jobs = multiprocessing.cpu_count() - except NotImplementedError: - n_jobs = 1 - if n_jobs > 1: - options.jobs = n_jobs - reporter = QueueReport - if options.quiet: - reporter = BaseQReport - if options.quiet == 1: - reporter = FileQReport - report = styleguide.init_report(reporter) - report.input_file = styleguide.input_file - styleguide.runner = report.task_queue.put - - return styleguide - - -def get_python_version(): - # The implementation isn't all that important. - try: - impl = platform.python_implementation() + " " - except AttributeError: # Python 2.5 - impl = '' - return '%s%s on %s' % (impl, platform.python_version(), platform.system()) - - -def make_stdin_get_value(original): - def stdin_get_value(): - if not hasattr(stdin_get_value, 'cached_stdin'): - value = original() - if sys.version_info < (3, 0): - stdin = io.BytesIO(value) - else: - stdin = io.StringIO(value) - stdin_get_value.cached_stdin = stdin - else: - stdin = stdin_get_value.cached_stdin - return stdin.getvalue() - - return stdin_get_value - - -pep8.stdin_get_value = make_stdin_get_value(pep8.stdin_get_value) diff --git a/contrib/flake8/exceptions.py b/contrib/flake8/exceptions.py new file mode 100644 index 0000000..13e8996 --- /dev/null +++ b/contrib/flake8/exceptions.py @@ -0,0 +1,129 @@ +"""Exception classes for all of Flake8.""" + + +class Flake8Exception(Exception): + """Plain Flake8 exception.""" + + pass + + +class EarlyQuit(Flake8Exception): + """Except raised when encountering a KeyboardInterrupt.""" + + pass + + +class ExecutionError(Flake8Exception): + """Exception raised during execution of Flake8.""" + + +class FailedToLoadPlugin(Flake8Exception): + """Exception raised when a plugin fails to load.""" + + FORMAT = 'Flake8 failed to load plugin "%(name)s" due to %(exc)s.' + + def __init__(self, *args, **kwargs): + """Initialize our FailedToLoadPlugin exception.""" + self.plugin = kwargs.pop('plugin') + self.ep_name = self.plugin.name + self.original_exception = kwargs.pop('exception') + super(FailedToLoadPlugin, self).__init__(*args, **kwargs) + + def __str__(self): + """Return a nice string for our exception.""" + return self.FORMAT % {'name': self.ep_name, + 'exc': self.original_exception} + + +class InvalidSyntax(Flake8Exception): + """Exception raised when tokenizing a file fails.""" + + def __init__(self, *args, **kwargs): + """Initialize our InvalidSyntax exception.""" + exception = kwargs.pop('exception', None) + self.original_exception = exception + self.error_message = '{0}: {1}'.format( + exception.__class__.__name__, + exception.args[0], + ) + self.error_code = 'E902' + self.line_number = 1 + self.column_number = 0 + super(InvalidSyntax, self).__init__( + self.error_message, + *args, + **kwargs + ) + + +class PluginRequestedUnknownParameters(Flake8Exception): + """The plugin requested unknown parameters.""" + + FORMAT = '"%(name)s" requested unknown parameters causing %(exc)s' + + def __init__(self, *args, **kwargs): + """Pop certain keyword arguments for initialization.""" + self.original_exception = kwargs.pop('exception') + self.plugin = kwargs.pop('plugin') + super(PluginRequestedUnknownParameters, self).__init__( + *args, + **kwargs + ) + + def __str__(self): + """Format our exception message.""" + return self.FORMAT % {'name': self.plugin['plugin_name'], + 'exc': self.original_exception} + + +class HookInstallationError(Flake8Exception): + """Parent exception for all hooks errors.""" + + pass + + +class GitHookAlreadyExists(HookInstallationError): + """Exception raised when the git pre-commit hook file already exists.""" + + def __init__(self, *args, **kwargs): + """Initialize the path attribute.""" + self.path = kwargs.pop('path') + super(GitHookAlreadyExists, self).__init__(*args, **kwargs) + + def __str__(self): + """Provide a nice message regarding the exception.""" + msg = ('The Git pre-commit hook ({0}) already exists. To convince ' + 'Flake8 to install the hook, please remove the existing ' + 'hook.') + return msg.format(self.path) + + +class MercurialHookAlreadyExists(HookInstallationError): + """Exception raised when a mercurial hook is already configured.""" + + hook_name = None + + def __init__(self, *args, **kwargs): + """Initialize the relevant attributes.""" + self.path = kwargs.pop('path') + self.value = kwargs.pop('value') + super(MercurialHookAlreadyExists, self).__init__(*args, **kwargs) + + def __str__(self): + """Return a nicely formatted string for these errors.""" + msg = ('The Mercurial {0} hook already exists with "{1}" in {2}. ' + 'To convince Flake8 to install the hook, please remove the ' + '{0} configuration from the [hooks] section of your hgrc.') + return msg.format(self.hook_name, self.value, self.path) + + +class MercurialCommitHookAlreadyExists(MercurialHookAlreadyExists): + """Exception raised when the hg commit hook is already configured.""" + + hook_name = 'commit' + + +class MercurialQRefreshHookAlreadyExists(MercurialHookAlreadyExists): + """Exception raised when the hg commit hook is already configured.""" + + hook_name = 'qrefresh' diff --git a/contrib/flake8/formatting/__init__.py b/contrib/flake8/formatting/__init__.py new file mode 100644 index 0000000..bf44801 --- /dev/null +++ b/contrib/flake8/formatting/__init__.py @@ -0,0 +1 @@ +"""Submodule containing the default formatters for Flake8.""" diff --git a/contrib/flake8/formatting/base.py b/contrib/flake8/formatting/base.py new file mode 100644 index 0000000..c4c67d5 --- /dev/null +++ b/contrib/flake8/formatting/base.py @@ -0,0 +1,199 @@ +"""The base class and interface for all formatting plugins.""" +from __future__ import print_function + + +class BaseFormatter(object): + """Class defining the formatter interface. + + .. attribute:: options + + The options parsed from both configuration files and the command-line. + + .. attribute:: filename + + If specified by the user, the path to store the results of the run. + + .. attribute:: output_fd + + Initialized when the :meth:`start` is called. This will be a file + object opened for writing. + + .. attribute:: newline + + The string to add to the end of a line. This is only used when the + output filename has been specified. + """ + + def __init__(self, options): + """Initialize with the options parsed from config and cli. + + This also calls a hook, :meth:`after_init`, so subclasses do not need + to call super to call this method. + + :param optparse.Values options: + User specified configuration parsed from both configuration files + and the command-line interface. + """ + self.options = options + self.filename = options.output_file + self.output_fd = None + self.newline = '\n' + self.after_init() + + def after_init(self): + """Initialize the formatter further.""" + pass + + def beginning(self, filename): + """Notify the formatter that we're starting to process a file. + + :param str filename: + The name of the file that Flake8 is beginning to report results + from. + """ + pass + + def finished(self, filename): + """Notify the formatter that we've finished processing a file. + + :param str filename: + The name of the file that Flake8 has finished reporting results + from. + """ + pass + + def start(self): + """Prepare the formatter to receive input. + + This defaults to initializing :attr:`output_fd` if :attr:`filename` + """ + if self.filename: + self.output_fd = open(self.filename, 'a') + + def handle(self, error): + """Handle an error reported by Flake8. + + This defaults to calling :meth:`format`, :meth:`show_source`, and + then :meth:`write`. To extend how errors are handled, override this + method. + + :param error: + This will be an instance of + :class:`~flake8.style_guide.Violation`. + :type error: + flake8.style_guide.Violation + """ + line = self.format(error) + source = self.show_source(error) + self.write(line, source) + + def format(self, error): + """Format an error reported by Flake8. + + This method **must** be implemented by subclasses. + + :param error: + This will be an instance of + :class:`~flake8.style_guide.Violation`. + :type error: + flake8.style_guide.Violation + :returns: + The formatted error string. + :rtype: + str + """ + raise NotImplementedError('Subclass of BaseFormatter did not implement' + ' format.') + + def show_statistics(self, statistics): + """Format and print the statistics.""" + for error_code in statistics.error_codes(): + stats_for_error_code = statistics.statistics_for(error_code) + statistic = next(stats_for_error_code) + count = statistic.count + count += sum(stat.count for stat in stats_for_error_code) + self._write('{count:<5} {error_code} {message}'.format( + count=count, + error_code=error_code, + message=statistic.message, + )) + + def show_benchmarks(self, benchmarks): + """Format and print the benchmarks.""" + # NOTE(sigmavirus24): The format strings are a little confusing, even + # to me, so here's a quick explanation: + # We specify the named value first followed by a ':' to indicate we're + # formatting the value. + # Next we use '<' to indicate we want the value left aligned. + # Then '10' is the width of the area. + # For floats, finally, we only want only want at most 3 digits after + # the decimal point to be displayed. This is the precision and it + # can not be specified for integers which is why we need two separate + # format strings. + float_format = '{value:<10.3} {statistic}'.format + int_format = '{value:<10} {statistic}'.format + for statistic, value in benchmarks: + if isinstance(value, int): + benchmark = int_format(statistic=statistic, value=value) + else: + benchmark = float_format(statistic=statistic, value=value) + self._write(benchmark) + + def show_source(self, error): + """Show the physical line generating the error. + + This also adds an indicator for the particular part of the line that + is reported as generating the problem. + + :param error: + This will be an instance of + :class:`~flake8.style_guide.Violation`. + :type error: + flake8.style_guide.Violation + :returns: + The formatted error string if the user wants to show the source. + If the user does not want to show the source, this will return + ``None``. + :rtype: + str + """ + if not self.options.show_source or error.physical_line is None: + return '' + + # Because column numbers are 1-indexed, we need to remove one to get + # the proper number of space characters. + pointer = (' ' * (error.column_number - 1)) + '^' + # Physical lines have a newline at the end, no need to add an extra + # one + return error.physical_line + pointer + + def _write(self, output): + """Handle logic of whether to use an output file or print().""" + if self.output_fd is not None: + self.output_fd.write(output + self.newline) + if self.output_fd is None or self.options.tee: + print(output) + + def write(self, line, source): + """Write the line either to the output file or stdout. + + This handles deciding whether to write to a file or print to standard + out for subclasses. Override this if you want behaviour that differs + from the default. + + :param str line: + The formatted string to print or write. + :param str source: + The source code that has been formatted and associated with the + line of output. + """ + if line: + self._write(line) + if source: + self._write(source) + + def stop(self): + """Clean up after reporting is finished.""" + if self.output_fd is not None: + self.output_fd.close() + self.output_fd = None diff --git a/contrib/flake8/formatting/default.py b/contrib/flake8/formatting/default.py new file mode 100644 index 0000000..8c91f9f --- /dev/null +++ b/contrib/flake8/formatting/default.py @@ -0,0 +1,88 @@ +"""Default formatting class for Flake8.""" +from flake8.formatting import base + + +class SimpleFormatter(base.BaseFormatter): + """Simple abstraction for Default and Pylint formatter commonality. + + Sub-classes of this need to define an ``error_format`` attribute in order + to succeed. The ``format`` method relies on that attribute and expects the + ``error_format`` string to use the old-style formatting strings with named + parameters: + + * code + * text + * path + * row + * col + + """ + + error_format = None + + def format(self, error): + """Format and write error out. + + If an output filename is specified, write formatted errors to that + file. Otherwise, print the formatted error to standard out. + """ + return self.error_format % { + "code": error.code, + "text": error.text, + "path": error.filename, + "row": error.line_number, + "col": error.column_number, + } + + +class Default(SimpleFormatter): + """Default formatter for Flake8. + + This also handles backwards compatibility for people specifying a custom + format string. + """ + + error_format = '%(path)s:%(row)d:%(col)d: %(code)s %(text)s' + + def after_init(self): + """Check for a custom format string.""" + if self.options.format.lower() != 'default': + self.error_format = self.options.format + + +class Pylint(SimpleFormatter): + """Pylint formatter for Flake8.""" + + error_format = '%(path)s:%(row)d: [%(code)s] %(text)s' + + +class FilenameOnly(SimpleFormatter): + """Only print filenames, e.g., flake8 -q.""" + + error_format = '%(path)s' + + def after_init(self): + """Initialize our set of filenames.""" + self.filenames_already_printed = set() + + def show_source(self, error): + """Do not include the source code.""" + pass + + def format(self, error): + """Ensure we only print each error once.""" + if error.filename not in self.filenames_already_printed: + self.filenames_already_printed.add(error.filename) + return super(FilenameOnly, self).format(error) + + +class Nothing(base.BaseFormatter): + """Print absolutely nothing.""" + + def format(self, error): + """Do nothing.""" + pass + + def show_source(self, error): + """Do not print the source.""" + pass diff --git a/contrib/flake8/hooks.py b/contrib/flake8/hooks.py deleted file mode 100644 index 14f844e..0000000 --- a/contrib/flake8/hooks.py +++ /dev/null @@ -1,297 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import with_statement -import os -import pep8 -import sys -import stat -from subprocess import Popen, PIPE -import shutil -import tempfile -try: - from configparser import ConfigParser -except ImportError: # Python 2 - from ConfigParser import ConfigParser - -from flake8 import compat -from flake8.engine import get_parser, get_style_guide -from flake8.main import DEFAULT_CONFIG - - -def git_hook(complexity=-1, strict=False, ignore=None, lazy=False): - """This is the function used by the git hook. - - :param int complexity: (optional), any value > 0 enables complexity - checking with mccabe - :param bool strict: (optional), if True, this returns the total number of - errors which will cause the hook to fail - :param str ignore: (optional), a comma-separated list of errors and - warnings to ignore - :param bool lazy: (optional), allows for the instances where you don't add - the files to the index before running a commit, e.g., git commit -a - :returns: total number of errors if strict is True, otherwise 0 - """ - gitcmd = "git diff-index --cached --name-only --diff-filter=ACMRTUXB HEAD" - if lazy: - # Catch all files, including those not added to the index - gitcmd = gitcmd.replace('--cached ', '') - - if hasattr(ignore, 'split'): - ignore = ignore.split(',') - - # Returns the exit code, list of files modified, list of error messages - _, files_modified, _ = run(gitcmd) - - # We only want to pass ignore and max_complexity if they differ from the - # defaults so that we don't override a local configuration file - options = {} - if ignore: - options['ignore'] = ignore - if complexity > -1: - options['max_complexity'] = complexity - - tmpdir = tempfile.mkdtemp() - - flake8_style = get_style_guide(config_file=DEFAULT_CONFIG, paths=['.'], - **options) - filepatterns = flake8_style.options.filename - - # Copy staged versions to temporary directory - files_to_check = [] - try: - for file_ in files_modified: - # get the staged version of the file - gitcmd_getstaged = "git show :%s" % file_ - _, out, _ = run(gitcmd_getstaged, raw_output=True, decode=False) - # write the staged version to temp dir with its full path to - # avoid overwriting files with the same name - dirname, filename = os.path.split(os.path.abspath(file_)) - prefix = os.path.commonprefix([dirname, tmpdir]) - dirname = compat.relpath(dirname, start=prefix) - dirname = os.path.join(tmpdir, dirname) - if not os.path.isdir(dirname): - os.makedirs(dirname) - - # check_files() only does this check if passed a dir; so we do it - if ((pep8.filename_match(file_, filepatterns) and - not flake8_style.excluded(file_))): - - filename = os.path.join(dirname, filename) - files_to_check.append(filename) - # write staged version of file to temporary directory - with open(filename, "wb") as fh: - fh.write(out) - - # Run the checks - report = flake8_style.check_files(files_to_check) - # remove temporary directory - finally: - shutil.rmtree(tmpdir, ignore_errors=True) - - if strict: - return report.total_errors - - return 0 - - -def hg_hook(ui, repo, **kwargs): - """This is the function executed directly by Mercurial as part of the - hook. This is never called directly by the user, so the parameters are - undocumented. If you would like to learn more about them, please feel free - to read the official Mercurial documentation. - """ - complexity = ui.config('flake8', 'complexity', default=-1) - strict = ui.configbool('flake8', 'strict', default=True) - ignore = ui.config('flake8', 'ignore', default=None) - config = ui.config('flake8', 'config', default=DEFAULT_CONFIG) - - paths = _get_files(repo, **kwargs) - - # We only want to pass ignore and max_complexity if they differ from the - # defaults so that we don't override a local configuration file - options = {} - if ignore: - options['ignore'] = ignore - if complexity > -1: - options['max_complexity'] = complexity - - flake8_style = get_style_guide(config_file=config, paths=['.'], - **options) - report = flake8_style.check_files(paths) - - if strict: - return report.total_errors - - return 0 - - -def run(command, raw_output=False, decode=True): - p = Popen(command.split(), stdout=PIPE, stderr=PIPE) - (stdout, stderr) = p.communicate() - # On python 3, subprocess.Popen returns bytes objects which expect - # endswith to be given a bytes object or a tuple of bytes but not native - # string objects. This is simply less mysterious than using b'.py' in the - # endswith method. That should work but might still fail horribly. - if decode: - if hasattr(stdout, 'decode'): - stdout = stdout.decode('utf-8') - if hasattr(stderr, 'decode'): - stderr = stderr.decode('utf-8') - if not raw_output: - stdout = [line.strip() for line in stdout.splitlines()] - stderr = [line.strip() for line in stderr.splitlines()] - return (p.returncode, stdout, stderr) - - -def _get_files(repo, **kwargs): - seen = set() - for rev in range(repo[kwargs['node']], len(repo)): - for file_ in repo[rev].files(): - file_ = os.path.join(repo.root, file_) - if file_ in seen or not os.path.exists(file_): - continue - seen.add(file_) - if file_.endswith('.py'): - yield file_ - - -def find_vcs(): - try: - _, git_dir, _ = run('git rev-parse --git-dir') - except OSError: - pass - else: - if git_dir and os.path.isdir(git_dir[0]): - if not os.path.isdir(os.path.join(git_dir[0], 'hooks')): - os.mkdir(os.path.join(git_dir[0], 'hooks')) - return os.path.join(git_dir[0], 'hooks', 'pre-commit') - try: - _, hg_dir, _ = run('hg root') - except OSError: - pass - else: - if hg_dir and os.path.isdir(hg_dir[0]): - return os.path.join(hg_dir[0], '.hg', 'hgrc') - return '' - - -def get_git_config(option, opt_type='', convert_type=True): - # type can be --bool, --int or an empty string - _, git_cfg_value, _ = run('git config --get %s %s' % (opt_type, option), - raw_output=True) - git_cfg_value = git_cfg_value.strip() - if not convert_type: - return git_cfg_value - if opt_type == '--bool': - git_cfg_value = git_cfg_value.lower() == 'true' - elif git_cfg_value and opt_type == '--int': - git_cfg_value = int(git_cfg_value) - return git_cfg_value - - -_params = { - 'FLAKE8_COMPLEXITY': '--int', - 'FLAKE8_STRICT': '--bool', - 'FLAKE8_IGNORE': '', - 'FLAKE8_LAZY': '--bool', -} - - -def get_git_param(option, default=''): - global _params - opt_type = _params[option] - param_value = get_git_config(option.lower().replace('_', '.'), - opt_type=opt_type, convert_type=False) - if param_value == '': - param_value = os.environ.get(option, default) - if opt_type == '--bool' and not isinstance(param_value, bool): - param_value = param_value.lower() == 'true' - elif param_value and opt_type == '--int': - param_value = int(param_value) - return param_value - - -git_hook_file = """#!/usr/bin/env python -import sys -from flake8.hooks import git_hook, get_git_param - -# `get_git_param` will retrieve configuration from your local git config and -# then fall back to using the environment variables that the hook has always -# supported. -# For example, to set the complexity, you'll need to do: -# git config flake8.complexity 10 -COMPLEXITY = get_git_param('FLAKE8_COMPLEXITY', 10) -STRICT = get_git_param('FLAKE8_STRICT', False) -IGNORE = get_git_param('FLAKE8_IGNORE', None) -LAZY = get_git_param('FLAKE8_LAZY', False) - -if __name__ == '__main__': - sys.exit(git_hook( - complexity=COMPLEXITY, - strict=STRICT, - ignore=IGNORE, - lazy=LAZY, - )) -""" - - -def _install_hg_hook(path): - getenv = os.environ.get - if not os.path.isfile(path): - # Make the file so we can avoid IOError's - open(path, 'w').close() - - c = ConfigParser() - c.readfp(open(path, 'r')) - if not c.has_section('hooks'): - c.add_section('hooks') - - if not c.has_option('hooks', 'commit'): - c.set('hooks', 'commit', 'python:flake8.hooks.hg_hook') - - if not c.has_option('hooks', 'qrefresh'): - c.set('hooks', 'qrefresh', 'python:flake8.hooks.hg_hook') - - if not c.has_section('flake8'): - c.add_section('flake8') - - if not c.has_option('flake8', 'complexity'): - c.set('flake8', 'complexity', str(getenv('FLAKE8_COMPLEXITY', 10))) - - if not c.has_option('flake8', 'strict'): - c.set('flake8', 'strict', getenv('FLAKE8_STRICT', False)) - - if not c.has_option('flake8', 'ignore'): - c.set('flake8', 'ignore', getenv('FLAKE8_IGNORE', '')) - - if not c.has_option('flake8', 'lazy'): - c.set('flake8', 'lazy', getenv('FLAKE8_LAZY', False)) - - with open(path, 'w') as fd: - c.write(fd) - - -def install_hook(): - vcs = find_vcs() - - if not vcs: - p = get_parser()[0] - sys.stderr.write('Error: could not find either a git or mercurial ' - 'directory. Please re-run this in a proper ' - 'repository.\n') - p.print_help() - sys.exit(1) - - status = 0 - if 'git' in vcs: - if os.path.exists(vcs): - sys.exit('Error: hook already exists (%s)' % vcs) - with open(vcs, 'w') as fd: - fd.write(git_hook_file) - # rwxr--r-- - os.chmod(vcs, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) - elif 'hg' in vcs: - _install_hg_hook(vcs) - else: - status = 1 - - sys.exit(status) diff --git a/contrib/flake8/main.py b/contrib/flake8/main.py deleted file mode 100644 index 297618a..0000000 --- a/contrib/flake8/main.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import re -import sys - -import setuptools - -from flake8.engine import get_parser, get_style_guide -from flake8.util import option_normalizer - -if sys.platform.startswith('win'): - DEFAULT_CONFIG = os.path.expanduser(r'~\.flake8') -else: - DEFAULT_CONFIG = os.path.join( - os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), - 'flake8' - ) - -EXTRA_IGNORE = [] - - -def main(): - """Parse options and run checks on Python source.""" - # Prepare - flake8_style = get_style_guide(parse_argv=True, config_file=DEFAULT_CONFIG) - options = flake8_style.options - - if options.install_hook: - from flake8.hooks import install_hook - install_hook() - - # Run the checkers - report = flake8_style.check_files() - - exit_code = print_report(report, flake8_style) - if exit_code > 0: - raise SystemExit(exit_code > 0) - - -def print_report(report, flake8_style): - # Print the final report - options = flake8_style.options - if options.statistics: - report.print_statistics() - if options.benchmark: - report.print_benchmark() - if report.total_errors: - if options.count: - sys.stderr.write(str(report.total_errors) + '\n') - if not options.exit_zero: - return 1 - return 0 - - -def check_file(path, ignore=(), complexity=-1): - """Checks a file using pep8 and pyflakes by default and mccabe - optionally. - - :param str path: path to the file to be checked - :param tuple ignore: (optional), error and warning codes to be ignored - :param int complexity: (optional), enables the mccabe check for values > 0 - """ - ignore = set(ignore).union(EXTRA_IGNORE) - flake8_style = get_style_guide( - config_file=DEFAULT_CONFIG, ignore=ignore, max_complexity=complexity) - return flake8_style.input_file(path) - - -def check_code(code, ignore=(), complexity=-1): - """Checks code using pep8 and pyflakes by default and mccabe optionally. - - :param str code: code to be checked - :param tuple ignore: (optional), error and warning codes to be ignored - :param int complexity: (optional), enables the mccabe check for values > 0 - """ - ignore = set(ignore).union(EXTRA_IGNORE) - flake8_style = get_style_guide( - config_file=DEFAULT_CONFIG, ignore=ignore, max_complexity=complexity) - return flake8_style.input_file(None, lines=code.splitlines(True)) - - -class Flake8Command(setuptools.Command): - """The :class:`Flake8Command` class is used by setuptools to perform - checks on registered modules. - """ - - description = "Run flake8 on modules registered in setuptools" - user_options = [] - - def initialize_options(self): - self.option_to_cmds = {} - parser = get_parser()[0] - for opt in parser.option_list: - cmd_name = opt._long_opts[0][2:] - option_name = cmd_name.replace('-', '_') - self.option_to_cmds[option_name] = cmd_name - setattr(self, option_name, None) - - def finalize_options(self): - self.options_dict = {} - for (option_name, cmd_name) in self.option_to_cmds.items(): - if option_name in ['help', 'verbose']: - continue - value = getattr(self, option_name) - if value is None: - continue - value = option_normalizer(value) - # Check if there's any values that need to be fixed. - if option_name == "include" and isinstance(value, str): - value = re.findall('[^,;\s]+', value) - - self.options_dict[option_name] = value - - def distribution_files(self): - if self.distribution.packages: - package_dirs = self.distribution.package_dir or {} - for package in self.distribution.packages: - pkg_dir = package - if package in package_dirs: - pkg_dir = package_dirs[package] - elif '' in package_dirs: - pkg_dir = package_dirs[''] + os.path.sep + pkg_dir - yield pkg_dir.replace('.', os.path.sep) - - if self.distribution.py_modules: - for filename in self.distribution.py_modules: - yield "%s.py" % filename - # Don't miss the setup.py file itself - yield "setup.py" - - def run(self): - # Prepare - paths = list(self.distribution_files()) - flake8_style = get_style_guide(config_file=DEFAULT_CONFIG, - paths=paths, - **self.options_dict) - - # Run the checkers - report = flake8_style.check_files() - exit_code = print_report(report, flake8_style) - if exit_code > 0: - raise SystemExit(exit_code > 0) diff --git a/contrib/flake8/main/__init__.py b/contrib/flake8/main/__init__.py new file mode 100644 index 0000000..d3aa1de --- /dev/null +++ b/contrib/flake8/main/__init__.py @@ -0,0 +1 @@ +"""Module containing the logic for the Flake8 entry-points.""" diff --git a/contrib/flake8/main/application.py b/contrib/flake8/main/application.py new file mode 100644 index 0000000..6c68305 --- /dev/null +++ b/contrib/flake8/main/application.py @@ -0,0 +1,410 @@ +"""Module containing the application logic for Flake8.""" +from __future__ import print_function + +import logging +import sys +import time + +import flake8 +from flake8 import checker +from flake8 import defaults +from flake8 import exceptions +from flake8 import style_guide +from flake8 import utils +from flake8.main import options +from flake8.options import aggregator, config +from flake8.options import manager +from flake8.plugins import manager as plugin_manager + +LOG = logging.getLogger(__name__) + + +class Application(object): + """Abstract our application into a class.""" + + def __init__(self, program='flake8', version=flake8.__version__): + # type: (str, str) -> NoneType + """Initialize our application. + + :param str program: + The name of the program/application that we're executing. + :param str version: + The version of the program/application we're executing. + """ + #: The timestamp when the Application instance was instantiated. + self.start_time = time.time() + #: The timestamp when the Application finished reported errors. + self.end_time = None + #: The name of the program being run + self.program = program + #: The version of the program being run + self.version = version + #: The instance of :class:`flake8.options.manager.OptionManager` used + #: to parse and handle the options and arguments passed by the user + self.option_manager = manager.OptionManager( + prog='flake8', version=flake8.__version__ + ) + options.register_default_options(self.option_manager) + #: The preliminary options parsed from CLI before plugins are loaded, + #: into a :class:`optparse.Values` instance + self.prelim_opts = None + #: The preliminary arguments parsed from CLI before plugins are loaded + self.prelim_args = None + #: The instance of :class:`flake8.options.config.ConfigFileFinder` + self.config_finder = None + + #: The :class:`flake8.options.config.LocalPlugins` found in config + self.local_plugins = None + #: The instance of :class:`flake8.plugins.manager.Checkers` + self.check_plugins = None + #: The instance of :class:`flake8.plugins.manager.Listeners` + self.listening_plugins = None + #: The instance of :class:`flake8.plugins.manager.ReportFormatters` + self.formatting_plugins = None + #: The user-selected formatter from :attr:`formatting_plugins` + self.formatter = None + #: The :class:`flake8.plugins.notifier.Notifier` for listening plugins + self.listener_trie = None + #: The :class:`flake8.style_guide.StyleGuide` built from the user's + #: options + self.guide = None + #: The :class:`flake8.checker.Manager` that will handle running all of + #: the checks selected by the user. + self.file_checker_manager = None + + #: The user-supplied options parsed into an instance of + #: :class:`optparse.Values` + self.options = None + #: The left over arguments that were not parsed by + #: :attr:`option_manager` + self.args = None + #: The number of errors, warnings, and other messages after running + #: flake8 and taking into account ignored errors and lines. + self.result_count = 0 + #: The total number of errors before accounting for ignored errors and + #: lines. + self.total_result_count = 0 + #: Whether or not something catastrophic happened and we should exit + #: with a non-zero status code + self.catastrophic_failure = False + + #: Whether the program is processing a diff or not + self.running_against_diff = False + #: The parsed diff information + self.parsed_diff = {} + + def parse_preliminary_options_and_args(self, argv=None): + """Get preliminary options and args from CLI, pre-plugin-loading. + + We need to know the values of a few standard options and args now, so + that we can find config files and configure logging. + + Since plugins aren't loaded yet, there may be some as-yet-unknown + options; we ignore those for now, they'll be parsed later when we do + real option parsing. + + Sets self.prelim_opts and self.prelim_args. + + :param list argv: + Command-line arguments passed in directly. + """ + # We haven't found or registered our plugins yet, so let's defer + # printing the version until we aggregate options from config files + # and the command-line. First, let's clone our arguments on the CLI, + # then we'll attempt to remove ``--version`` so that we can avoid + # triggering the "version" action in optparse. If it's not there, we + # do not need to worry and we can continue. If it is, we successfully + # defer printing the version until just a little bit later. + # Similarly we have to defer printing the help text until later. + args = (argv or sys.argv)[:] + try: + args.remove('--version') + except ValueError: + pass + try: + args.remove('--help') + except ValueError: + pass + try: + args.remove('-h') + except ValueError: + pass + + opts, args = self.option_manager.parse_known_args(args) + # parse_known_args includes program name and unknown options as args + args = [a for a in args[1:] if not a.startswith('-')] + self.prelim_opts, self.prelim_args = opts, args + + def exit(self): + # type: () -> NoneType + """Handle finalization and exiting the program. + + This should be the last thing called on the application instance. It + will check certain options and exit appropriately. + """ + if self.options.count: + print(self.result_count) + + if not self.options.exit_zero: + raise SystemExit((self.result_count > 0) or + self.catastrophic_failure) + + def make_config_finder(self): + """Make our ConfigFileFinder based on preliminary opts and args.""" + if self.config_finder is None: + extra_config_files = utils.normalize_paths( + self.prelim_opts.append_config) + self.config_finder = config.ConfigFileFinder( + self.option_manager.program_name, + self.prelim_args, + extra_config_files, + ) + + def find_plugins(self): + # type: () -> NoneType + """Find and load the plugins for this application. + + If :attr:`check_plugins`, :attr:`listening_plugins`, or + :attr:`formatting_plugins` are ``None`` then this method will update + them with the appropriate plugin manager instance. Given the expense + of finding plugins (via :mod:`pkg_resources`) we want this to be + idempotent and so only update those attributes if they are ``None``. + """ + if self.local_plugins is None: + self.local_plugins = config.get_local_plugins( + self.config_finder, + self.prelim_opts.config, + self.prelim_opts.isolated, + ) + + if self.check_plugins is None: + self.check_plugins = plugin_manager.Checkers( + self.local_plugins.extension) + + if self.listening_plugins is None: + self.listening_plugins = plugin_manager.Listeners() + + if self.formatting_plugins is None: + self.formatting_plugins = plugin_manager.ReportFormatters( + self.local_plugins.report) + + self.check_plugins.load_plugins() + self.listening_plugins.load_plugins() + self.formatting_plugins.load_plugins() + + def register_plugin_options(self): + # type: () -> NoneType + """Register options provided by plugins to our option manager.""" + self.check_plugins.register_options(self.option_manager) + self.check_plugins.register_plugin_versions(self.option_manager) + self.listening_plugins.register_options(self.option_manager) + self.formatting_plugins.register_options(self.option_manager) + + def parse_configuration_and_cli(self, argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Parse configuration files and the CLI options. + + :param list argv: + Command-line arguments passed in directly. + """ + if self.options is None and self.args is None: + self.options, self.args = aggregator.aggregate_options( + self.option_manager, self.config_finder, argv + ) + + self.running_against_diff = self.options.diff + if self.running_against_diff: + self.parsed_diff = utils.parse_unified_diff() + if not self.parsed_diff: + self.exit() + + self.options._running_from_vcs = False + + self.check_plugins.provide_options(self.option_manager, self.options, + self.args) + self.listening_plugins.provide_options(self.option_manager, + self.options, + self.args) + self.formatting_plugins.provide_options(self.option_manager, + self.options, + self.args) + + def formatter_for(self, formatter_plugin_name): + """Retrieve the formatter class by plugin name.""" + try: + default_formatter = self.formatting_plugins['default'] + except KeyError: + raise exceptions.ExecutionError( + "The 'default' Flake8 formatting plugin is unavailable. " + "This usually indicates that your setuptools is too old. " + "Please upgrade setuptools. If that does not fix the issue" + " please file an issue." + ) + + formatter_plugin = self.formatting_plugins.get(formatter_plugin_name) + if formatter_plugin is None: + LOG.warning( + '"%s" is an unknown formatter. Falling back to default.', + formatter_plugin_name, + ) + formatter_plugin = default_formatter + + return formatter_plugin.execute + + def make_formatter(self, formatter_class=None): + # type: () -> NoneType + """Initialize a formatter based on the parsed options.""" + if self.formatter is None: + format_plugin = self.options.format + if 1 <= self.options.quiet < 2: + format_plugin = 'quiet-filename' + elif 2 <= self.options.quiet: + format_plugin = 'quiet-nothing' + + if formatter_class is None: + formatter_class = self.formatter_for(format_plugin) + + self.formatter = formatter_class(self.options) + + def make_notifier(self): + # type: () -> NoneType + """Initialize our listener Notifier.""" + if self.listener_trie is None: + self.listener_trie = self.listening_plugins.build_notifier() + + def make_guide(self): + # type: () -> NoneType + """Initialize our StyleGuide.""" + if self.guide is None: + self.guide = style_guide.StyleGuide( + self.options, self.listener_trie, self.formatter + ) + + if self.running_against_diff: + self.guide.add_diff_ranges(self.parsed_diff) + + def make_file_checker_manager(self): + # type: () -> NoneType + """Initialize our FileChecker Manager.""" + if self.file_checker_manager is None: + self.file_checker_manager = checker.Manager( + style_guide=self.guide, + arguments=self.args, + checker_plugins=self.check_plugins, + ) + + def run_checks(self, files=None): + # type: (Union[List[str], NoneType]) -> NoneType + """Run the actual checks with the FileChecker Manager. + + This method encapsulates the logic to make a + :class:`~flake8.checker.Manger` instance run the checks it is + managing. + + :param list files: + List of filenames to process + """ + if self.running_against_diff: + files = sorted(self.parsed_diff) + self.file_checker_manager.start(files) + self.file_checker_manager.run() + LOG.info('Finished running') + self.file_checker_manager.stop() + self.end_time = time.time() + + def report_benchmarks(self): + """Aggregate, calculate, and report benchmarks for this run.""" + if not self.options.benchmark: + return + + time_elapsed = self.end_time - self.start_time + statistics = [('seconds elapsed', time_elapsed)] + add_statistic = statistics.append + for statistic in (defaults.STATISTIC_NAMES + ('files',)): + value = self.file_checker_manager.statistics[statistic] + total_description = 'total ' + statistic + ' processed' + add_statistic((total_description, value)) + per_second_description = statistic + ' processed per second' + add_statistic((per_second_description, int(value / time_elapsed))) + + self.formatter.show_benchmarks(statistics) + + def report_errors(self): + # type: () -> NoneType + """Report all the errors found by flake8 3.0. + + This also updates the :attr:`result_count` attribute with the total + number of errors, warnings, and other messages found. + """ + LOG.info('Reporting errors') + results = self.file_checker_manager.report() + self.total_result_count, self.result_count = results + LOG.info('Found a total of %d violations and reported %d', + self.total_result_count, self.result_count) + + def report_statistics(self): + """Aggregate and report statistics from this run.""" + if not self.options.statistics: + return + + self.formatter.show_statistics(self.guide.stats) + + def initialize(self, argv): + # type: () -> NoneType + """Initialize the application to be run. + + This finds the plugins, registers their options, and parses the + command-line arguments. + """ + # NOTE(sigmavirus24): When updating this, make sure you also update + # our legacy API calls to these same methods. + self.parse_preliminary_options_and_args(argv) + flake8.configure_logging( + self.prelim_opts.verbose, self.prelim_opts.output_file) + self.make_config_finder() + self.find_plugins() + self.register_plugin_options() + self.parse_configuration_and_cli(argv) + self.make_formatter() + self.make_notifier() + self.make_guide() + self.make_file_checker_manager() + + def report(self): + """Report errors, statistics, and benchmarks.""" + self.formatter.start() + self.report_errors() + self.report_statistics() + self.report_benchmarks() + self.formatter.stop() + + def _run(self, argv): + # type: (Union[NoneType, List[str]]) -> NoneType + self.initialize(argv) + self.run_checks() + self.report() + + def run(self, argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Run our application. + + This method will also handle KeyboardInterrupt exceptions for the + entirety of the flake8 application. If it sees a KeyboardInterrupt it + will forcibly clean up the :class:`~flake8.checker.Manager`. + """ + try: + self._run(argv) + except KeyboardInterrupt as exc: + print('... stopped') + LOG.critical('Caught keyboard interrupt from user') + LOG.exception(exc) + self.file_checker_manager._force_cleanup() + self.catastrophic_failure = True + except exceptions.ExecutionError as exc: + print('There was a critical error during execution of Flake8:') + print(exc.message) + LOG.exception(exc) + self.catastrophic_failure = True + except exceptions.EarlyQuit: + self.catastrophic_failure = True + print('... stopped while processing files') diff --git a/contrib/flake8/main/cli.py b/contrib/flake8/main/cli.py new file mode 100644 index 0000000..8ba6288 --- /dev/null +++ b/contrib/flake8/main/cli.py @@ -0,0 +1,17 @@ +"""Command-line implementation of flake8.""" +from flake8.main import application + + +def main(argv=None): + # type: (Union[NoneType, List[str]]) -> NoneType + """Execute the main bit of the application. + + This handles the creation of an instance of :class:`Application`, runs it, + and then exits the application. + + :param list argv: + The arguments to be passed to the application for parsing. + """ + app = application.Application() + app.run(argv) + app.exit() diff --git a/contrib/flake8/main/debug.py b/contrib/flake8/main/debug.py new file mode 100644 index 0000000..ca3827e --- /dev/null +++ b/contrib/flake8/main/debug.py @@ -0,0 +1,68 @@ +"""Module containing the logic for our debugging logic.""" +from __future__ import print_function + +import json +import platform + +import setuptools + + +def print_information(option, option_string, value, parser, + option_manager=None): + """Print debugging information used in bug reports. + + :param option: + The optparse Option instance. + :type option: + optparse.Option + :param str option_string: + The option name + :param value: + The value passed to the callback parsed from the command-line + :param parser: + The optparse OptionParser instance + :type parser: + optparse.OptionParser + :param option_manager: + The Flake8 OptionManager instance. + :type option_manager: + flake8.options.manager.OptionManager + """ + if not option_manager.registered_plugins: + # NOTE(sigmavirus24): Flake8 parses options twice. The first time, we + # will not have any registered plugins. We can skip this one and only + # take action on the second time we're called. + return + print(json.dumps(information(option_manager), indent=2, sort_keys=True)) + raise SystemExit(False) + + +def information(option_manager): + """Generate the information to be printed for the bug report.""" + return { + 'version': option_manager.version, + 'plugins': plugins_from(option_manager), + 'dependencies': dependencies(), + 'platform': { + 'python_implementation': platform.python_implementation(), + 'python_version': platform.python_version(), + 'system': platform.system(), + }, + } + + +def plugins_from(option_manager): + """Generate the list of plugins installed.""" + return [ + { + 'plugin': plugin.name, + 'version': plugin.version, + 'is_local': plugin.local, + } + for plugin in sorted(option_manager.registered_plugins) + ] + + +def dependencies(): + """Generate the list of dependencies we care about.""" + return [{'dependency': 'setuptools', 'version': setuptools.__version__}] diff --git a/contrib/flake8/main/git.py b/contrib/flake8/main/git.py new file mode 100644 index 0000000..ad55100 --- /dev/null +++ b/contrib/flake8/main/git.py @@ -0,0 +1,244 @@ +"""Module containing the main git hook interface and helpers. + +.. autofunction:: hook +.. autofunction:: install + +""" +import contextlib +import os +import os.path +import shutil +import stat +import subprocess +import sys +import tempfile + +from flake8 import defaults +from flake8 import exceptions + +__all__ = ('hook', 'install') + + +def hook(lazy=False, strict=False): + """Execute Flake8 on the files in git's index. + + Determine which files are about to be committed and run Flake8 over them + to check for violations. + + :param bool lazy: + Find files not added to the index prior to committing. This is useful + if you frequently use ``git commit -a`` for example. This defaults to + False since it will otherwise include files not in the index. + :param bool strict: + If True, return the total number of errors/violations found by Flake8. + This will cause the hook to fail. + :returns: + Total number of errors found during the run. + :rtype: + int + """ + # NOTE(sigmavirus24): Delay import of application until we need it. + from flake8.main import application + app = application.Application() + with make_temporary_directory() as tempdir: + filepaths = list(copy_indexed_files_to(tempdir, lazy)) + app.initialize(['.']) + app.options.exclude = update_excludes(app.options.exclude, tempdir) + app.options._running_from_vcs = True + # Apparently there are times when there are no files to check (e.g., + # when amending a commit). In those cases, let's not try to run checks + # against nothing. + if filepaths: + app.run_checks(filepaths) + + # If there were files to check, update their paths and report the errors + if filepaths: + update_paths(app.file_checker_manager, tempdir) + app.report_errors() + + if strict: + return app.result_count + return 0 + + +def install(): + """Install the git hook script. + + This searches for the ``.git`` directory and will install an executable + pre-commit python script in the hooks sub-directory if one does not + already exist. + + It will also print a message to stdout about how to configure the hook. + + :returns: + True if successful, False if the git directory doesn't exist. + :rtype: + bool + :raises: + flake8.exceptions.GitHookAlreadyExists + """ + git_directory = find_git_directory() + if git_directory is None or not os.path.exists(git_directory): + return False + + hooks_directory = os.path.join(git_directory, 'hooks') + if not os.path.exists(hooks_directory): + os.mkdir(hooks_directory) + + pre_commit_file = os.path.abspath( + os.path.join(hooks_directory, 'pre-commit') + ) + if os.path.exists(pre_commit_file): + raise exceptions.GitHookAlreadyExists( + 'File already exists', + path=pre_commit_file, + ) + + executable = get_executable() + + with open(pre_commit_file, 'w') as fd: + fd.write(_HOOK_TEMPLATE.format(executable=executable)) + + # NOTE(sigmavirus24): The following sets: + # - read, write, and execute permissions for the owner + # - read permissions for people in the group + # - read permissions for other people + # The owner needs the file to be readable, writable, and executable + # so that git can actually execute it as a hook. + pre_commit_permissions = stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH + os.chmod(pre_commit_file, pre_commit_permissions) + + print('git pre-commit hook installed, for configuration options see') + print('http://flake8.pycqa.org/en/latest/user/using-hooks.html') + + return True + + +def get_executable(): + if sys.executable is not None: + return sys.executable + return '/usr/bin/env python' + + +def find_git_directory(): + rev_parse = piped_process(['git', 'rev-parse', '--git-dir']) + + (stdout, _) = rev_parse.communicate() + stdout = to_text(stdout) + + if rev_parse.returncode == 0: + return stdout.strip() + return None + + +def copy_indexed_files_to(temporary_directory, lazy): + modified_files = find_modified_files(lazy) + for filename in modified_files: + contents = get_staged_contents_from(filename) + yield copy_file_to(temporary_directory, filename, contents) + + +def copy_file_to(destination_directory, filepath, contents): + directory, filename = os.path.split(os.path.abspath(filepath)) + temporary_directory = make_temporary_directory_from(destination_directory, + directory) + if not os.path.exists(temporary_directory): + os.makedirs(temporary_directory) + temporary_filepath = os.path.join(temporary_directory, filename) + with open(temporary_filepath, 'wb') as fd: + fd.write(contents) + return temporary_filepath + + +def make_temporary_directory_from(destination, directory): + prefix = os.path.commonprefix([directory, destination]) + common_directory_path = os.path.relpath(directory, start=prefix) + return os.path.join(destination, common_directory_path) + + +def find_modified_files(lazy): + diff_index_cmd = [ + 'git', 'diff-index', '--cached', '--name-only', + '--diff-filter=ACMRTUXB', 'HEAD' + ] + if lazy: + diff_index_cmd.remove('--cached') + + diff_index = piped_process(diff_index_cmd) + (stdout, _) = diff_index.communicate() + stdout = to_text(stdout) + return stdout.splitlines() + + +def get_staged_contents_from(filename): + git_show = piped_process(['git', 'show', ':{0}'.format(filename)]) + (stdout, _) = git_show.communicate() + return stdout + + +@contextlib.contextmanager +def make_temporary_directory(): + temporary_directory = tempfile.mkdtemp() + yield temporary_directory + shutil.rmtree(temporary_directory, ignore_errors=True) + + +def to_text(string): + """Ensure that the string is text.""" + if callable(getattr(string, 'decode', None)): + return string.decode('utf-8') + return string + + +def piped_process(command): + return subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def git_config_for(parameter): + config = piped_process(['git', 'config', '--get', '--bool', parameter]) + (stdout, _) = config.communicate() + return to_text(stdout).strip() + + +def config_for(parameter): + environment_variable = 'flake8_{0}'.format(parameter).upper() + git_variable = 'flake8.{0}'.format(parameter) + value = os.environ.get(environment_variable, git_config_for(git_variable)) + return value.lower() in defaults.TRUTHY_VALUES + + +def update_excludes(exclude_list, temporary_directory_path): + return [ + (temporary_directory_path + pattern) + if os.path.isabs(pattern) else pattern + for pattern in exclude_list + ] + + +def update_paths(checker_manager, temp_prefix): + temp_prefix_length = len(temp_prefix) + for checker in checker_manager.checkers: + filename = checker.display_name + if filename.startswith(temp_prefix): + checker.display_name = os.path.relpath( + filename[temp_prefix_length:] + ) + + +_HOOK_TEMPLATE = """#!{executable} +import sys + +from flake8.main import git + +if __name__ == '__main__': + sys.exit( + git.hook( + strict=git.config_for('strict'), + lazy=git.config_for('lazy'), + ) + ) +""" diff --git a/contrib/flake8/main/mercurial.py b/contrib/flake8/main/mercurial.py new file mode 100644 index 0000000..344c9f7 --- /dev/null +++ b/contrib/flake8/main/mercurial.py @@ -0,0 +1,149 @@ +"""Module containing the main mecurial hook interface and helpers. + +.. autofunction:: hook +.. autofunction:: install + +""" +import configparser +import os +import subprocess + +from flake8 import exceptions as exc + +__all__ = ('hook', 'install') + + +def hook(ui, repo, **kwargs): + """Execute Flake8 on the repository provided by Mercurial. + + To understand the parameters read more of the Mercurial documentation + around Hooks: https://www.mercurial-scm.org/wiki/Hook. + + We avoid using the ``ui`` attribute because it can cause issues with + the GPL license tha Mercurial is under. We don't import it, but we + avoid using it all the same. + """ + from flake8.main import application + hgrc = find_hgrc(create_if_missing=False) + if hgrc is None: + print('Cannot locate your root mercurial repository.') + raise SystemExit(True) + + hgconfig = configparser_for(hgrc) + strict = hgconfig.get('flake8', 'strict', fallback=True) + + filenames = list(get_filenames_from(repo, kwargs)) + + app = application.Application() + app.initialize(filenames) + app.options._running_from_vcs = True + app.run_checks() + app.report() + + if strict: + return app.result_count + return 0 + + +def install(): + """Ensure that the mercurial hooks are installed. + + This searches for the ``.hg/hgrc`` configuration file and will add commit + and qrefresh hooks to it, if they do not already exist. + + It will also print a message to stdout about how to configure the hook. + + :returns: + True if successful, False if the ``.hg/hgrc`` file doesn't exist. + :rtype: + bool + :raises: + flake8.exceptions.MercurialCommitHookAlreadyExists + :raises: + flake8.exceptions.MercurialQRefreshHookAlreadyExists + """ + hgrc = find_hgrc(create_if_missing=True) + if hgrc is None: + return False + + hgconfig = configparser_for(hgrc) + + if not hgconfig.has_section('hooks'): + hgconfig.add_section('hooks') + + if hgconfig.has_option('hooks', 'commit'): + raise exc.MercurialCommitHookAlreadyExists( + path=hgrc, + value=hgconfig.get('hooks', 'commit'), + ) + + if hgconfig.has_option('hooks', 'qrefresh'): + raise exc.MercurialQRefreshHookAlreadyExists( + path=hgrc, + value=hgconfig.get('hooks', 'qrefresh'), + ) + + hgconfig.set('hooks', 'commit', 'python:flake8.main.mercurial.hook') + hgconfig.set('hooks', 'qrefresh', 'python:flake8.main.mercurial.hook') + + if not hgconfig.has_section('flake8'): + hgconfig.add_section('flake8') + + if not hgconfig.has_option('flake8', 'strict'): + hgconfig.set('flake8', 'strict', False) + + with open(hgrc, 'w') as fd: + hgconfig.write(fd) + + print('mercurial hooks installed, for configuration options see') + print('http://flake8.pycqa.org/en/latest/user/using-hooks.html') + + return True + + +def get_filenames_from(repository, kwargs): + seen_filenames = set() + node = kwargs['node'] + for revision in range(repository[node], len(repository)): + for filename in repository[revision].files(): + full_filename = os.path.join(repository.root, filename) + have_seen_filename = full_filename in seen_filenames + filename_does_not_exist = not os.path.exists(full_filename) + if have_seen_filename or filename_does_not_exist: + continue + + seen_filenames.add(full_filename) + if full_filename.endswith('.py'): + yield full_filename + + +def find_hgrc(create_if_missing=False): + root = subprocess.Popen( + ['hg', 'root'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + (hg_directory, _) = root.communicate() + if callable(getattr(hg_directory, 'decode', None)): + hg_directory = hg_directory.decode('utf-8') + + if not os.path.isdir(hg_directory): + return None + + hgrc = os.path.abspath( + os.path.join(hg_directory, '.hg', 'hgrc') + ) + if not os.path.exists(hgrc): + if create_if_missing: + open(hgrc, 'w').close() + else: + return None + + return hgrc + + +def configparser_for(path): + parser = configparser.ConfigParser(interpolation=None) + parser.read(path) + return parser diff --git a/contrib/flake8/main/options.py b/contrib/flake8/main/options.py new file mode 100644 index 0000000..131b714 --- /dev/null +++ b/contrib/flake8/main/options.py @@ -0,0 +1,218 @@ +"""Contains the logic for all of the default options for Flake8.""" +from flake8 import defaults +from flake8.main import debug +from flake8.main import vcs + + +def register_default_options(option_manager): + """Register the default options on our OptionManager. + + The default options include: + + - ``-v``/``--verbose`` + - ``-q``/``--quiet`` + - ``--count`` + - ``--diff`` + - ``--exclude`` + - ``--filename`` + - ``--format`` + - ``--hang-closing`` + - ``--ignore`` + - ``--max-line-length`` + - ``--select`` + - ``--disable-noqa`` + - ``--show-source`` + - ``--statistics`` + - ``--enable-extensions`` + - ``--exit-zero`` + - ``-j``/``--jobs`` + - ``--output-file`` + - ``--tee`` + - ``--append-config`` + - ``--config`` + - ``--isolated`` + - ``--benchmark`` + - ``--bug-report`` + """ + add_option = option_manager.add_option + + # pep8 options + add_option( + '-v', '--verbose', default=0, action='count', + parse_from_config=True, + help='Print more information about what is happening in flake8.' + ' This option is repeatable and will increase verbosity each ' + 'time it is repeated.', + ) + add_option( + '-q', '--quiet', default=0, action='count', + parse_from_config=True, + help='Report only file names, or nothing. This option is repeatable.', + ) + + add_option( + '--count', action='store_true', parse_from_config=True, + help='Print total number of errors and warnings to standard error and' + ' set the exit code to 1 if total is not empty.', + ) + + add_option( + '--diff', action='store_true', + help='Report changes only within line number ranges in the unified ' + 'diff provided on standard in by the user.', + ) + + add_option( + '--exclude', metavar='patterns', default=','.join(defaults.EXCLUDE), + comma_separated_list=True, parse_from_config=True, + normalize_paths=True, + help='Comma-separated list of files or directories to exclude.' + ' (Default: %default)', + ) + + add_option( + '--filename', metavar='patterns', default='*.py', + parse_from_config=True, comma_separated_list=True, + help='Only check for filenames matching the patterns in this comma-' + 'separated list. (Default: %default)', + ) + + add_option( + '--stdin-display-name', default='stdin', + help='The name used when reporting errors from code passed via stdin.' + ' This is useful for editors piping the file contents to flake8.' + ' (Default: %default)', + ) + + # TODO(sigmavirus24): Figure out --first/--repeat + + # NOTE(sigmavirus24): We can't use choices for this option since users can + # freely provide a format string and that will break if we restrict their + # choices. + add_option( + '--format', metavar='format', default='default', + parse_from_config=True, + help='Format errors according to the chosen formatter.', + ) + + add_option( + '--hang-closing', action='store_true', parse_from_config=True, + help='Hang closing bracket instead of matching indentation of opening' + " bracket's line.", + ) + + add_option( + '--ignore', metavar='errors', default=','.join(defaults.IGNORE), + parse_from_config=True, comma_separated_list=True, + help='Comma-separated list of errors and warnings to ignore (or skip).' + ' For example, ``--ignore=E4,E51,W234``. (Default: %default)', + ) + + add_option( + '--max-line-length', type='int', metavar='n', + default=defaults.MAX_LINE_LENGTH, parse_from_config=True, + help='Maximum allowed line length for the entirety of this run. ' + '(Default: %default)', + ) + + add_option( + '--select', metavar='errors', default=','.join(defaults.SELECT), + parse_from_config=True, comma_separated_list=True, + help='Comma-separated list of errors and warnings to enable.' + ' For example, ``--select=E4,E51,W234``. (Default: %default)', + ) + + add_option( + '--disable-noqa', default=False, parse_from_config=True, + action='store_true', + help='Disable the effect of "# noqa". This will report errors on ' + 'lines with "# noqa" at the end.' + ) + + # TODO(sigmavirus24): Decide what to do about --show-pep8 + + add_option( + '--show-source', action='store_true', parse_from_config=True, + help='Show the source generate each error or warning.', + ) + + add_option( + '--statistics', action='store_true', parse_from_config=True, + help='Count errors and warnings.', + ) + + # Flake8 options + add_option( + '--enable-extensions', default='', parse_from_config=True, + comma_separated_list=True, type='string', + help='Enable plugins and extensions that are otherwise disabled ' + 'by default', + ) + + add_option( + '--exit-zero', action='store_true', + help='Exit with status code "0" even if there are errors.', + ) + + add_option( + '--install-hook', action='callback', type='choice', + choices=vcs.choices(), callback=vcs.install, + help='Install a hook that is run prior to a commit for the supported ' + 'version control system.' + ) + + add_option( + '-j', '--jobs', type='string', default='auto', parse_from_config=True, + help='Number of subprocesses to use to run checks in parallel. ' + 'This is ignored on Windows. The default, "auto", will ' + 'auto-detect the number of processors available to use.' + ' (Default: %default)', + ) + + add_option( + '--output-file', default=None, type='string', parse_from_config=True, + # callback=callbacks.redirect_stdout, + help='Redirect report to a file.', + ) + + add_option( + '--tee', default=False, parse_from_config=True, action='store_true', + help='Write to stdout and output-file.', + ) + + # Config file options + + add_option( + '--append-config', action='append', + help='Provide extra config files to parse in addition to the files ' + 'found by Flake8 by default. These files are the last ones read ' + 'and so they take the highest precedence when multiple files ' + 'provide the same option.', + ) + + add_option( + '--config', default=None, + help='Path to the config file that will be the authoritative config ' + 'source. This will cause Flake8 to ignore all other ' + 'configuration files.' + ) + + add_option( + '--isolated', default=False, action='store_true', + help='Ignore all found configuration files.', + ) + + # Benchmarking + + add_option( + '--benchmark', default=False, action='store_true', + help='Print benchmark information about this run of Flake8', + ) + + # Debugging + + add_option( + '--bug-report', action='callback', callback=debug.print_information, + callback_kwargs={'option_manager': option_manager}, + help='Print information necessary when preparing a bug report', + ) diff --git a/contrib/flake8/main/setuptools_command.py b/contrib/flake8/main/setuptools_command.py new file mode 100644 index 0000000..a5d23a5 --- /dev/null +++ b/contrib/flake8/main/setuptools_command.py @@ -0,0 +1,103 @@ +"""The logic for Flake8's integration with setuptools.""" +import os + +import setuptools + +from flake8.main import application as app + +UNSET = object() + + +class Flake8(setuptools.Command): + """Run Flake8 via setuptools/distutils for registered modules.""" + + description = 'Run Flake8 on modules registered in setup.py' + # NOTE(sigmavirus24): If we populated this with a list of tuples, users + # could do something like ``python setup.py flake8 --ignore=E123,E234`` + # but we would have to redefine it and we can't define it dynamically. + # Since I refuse to copy-and-paste the options here or maintain two lists + # of options, and since this will break when users use plugins that + # provide command-line options, we are leaving this empty. If users want + # to configure this command, they can do so through config files. + user_options = [] + + def initialize_options(self): + """Override this method to initialize our application.""" + self.flake8 = app.Application() + self.flake8.initialize([]) + options = self.flake8.option_manager.options + for option in options: + if option.parse_from_config: + setattr(self, option.config_name, UNSET) + + def finalize_options(self): + """Override this to parse the parameters.""" + options = self.flake8.option_manager.options + for option in options: + if option.parse_from_config: + name = option.config_name + value = getattr(self, name, UNSET) + if value is UNSET: + continue + setattr(self.flake8.options, + name, + option.normalize_from_setuptools(value)) + + def package_files(self): + """Collect the files/dirs included in the registered modules.""" + seen_package_directories = () + directories = self.distribution.package_dir or {} + empty_directory_exists = '' in directories + packages = self.distribution.packages or [] + for package in packages: + package_directory = package + if package in directories: + package_directory = directories[package] + elif empty_directory_exists: + package_directory = os.path.join(directories[''], + package_directory) + + # NOTE(sigmavirus24): Do not collect submodules, e.g., + # if we have: + # - flake8/ + # - flake8/plugins/ + # Flake8 only needs ``flake8/`` to be provided. It will + # recurse on its own. + if package_directory.startswith(seen_package_directories): + continue + + seen_package_directories += (package_directory + '.',) + yield package_directory + + def module_files(self): + """Collect the files listed as py_modules.""" + modules = self.distribution.py_modules or [] + filename_from = '{0}.py'.format + for module in modules: + yield filename_from(module) + + def distribution_files(self): + """Collect package and module files.""" + for package in self.package_files(): + yield package + + for module in self.module_files(): + yield module + + yield 'setup.py' + + def run(self): + """Run the Flake8 application.""" + self.flake8.run_checks(list(self.distribution_files())) + self.flake8.formatter.start() + self.flake8.report_errors() + self.flake8.report_statistics() + self.flake8.report_benchmarks() + self.flake8.formatter.stop() + try: + self.flake8.exit() + except SystemExit as e: + # Cause system exit only if exit code is not zero (terminates + # other possibly remaining/pending setuptools commands). + if e.code: + raise diff --git a/contrib/flake8/main/vcs.py b/contrib/flake8/main/vcs.py new file mode 100644 index 0000000..207d26c --- /dev/null +++ b/contrib/flake8/main/vcs.py @@ -0,0 +1,39 @@ +"""Module containing some of the logic for our VCS installation logic.""" +from flake8 import exceptions as exc +from flake8.main import git +from flake8.main import mercurial + + +# NOTE(sigmavirus24): In the future, we may allow for VCS hooks to be defined +# as plugins, e.g., adding a flake8.vcs entry-point. In that case, this +# dictionary should disappear, and this module might contain more code for +# managing those bits (in conjuntion with flake8.plugins.manager). +_INSTALLERS = { + 'git': git.install, + 'mercurial': mercurial.install, +} + + +def install(option, option_string, value, parser): + """Determine which version control hook to install. + + For more information about the callback signature, see: + https://docs.python.org/2/library/optparse.html#optparse-option-callbacks + """ + installer = _INSTALLERS.get(value) + errored = False + successful = False + try: + successful = installer() + except exc.HookInstallationError as hook_error: + print(str(hook_error)) + errored = True + + if not successful: + print('Could not find the {0} directory'.format(value)) + raise SystemExit(not successful and errored) + + +def choices(): + """Return the list of VCS choices.""" + return list(_INSTALLERS) diff --git a/contrib/flake8/options/__init__.py b/contrib/flake8/options/__init__.py new file mode 100644 index 0000000..cc20daa --- /dev/null +++ b/contrib/flake8/options/__init__.py @@ -0,0 +1,12 @@ +"""Package containing the option manager and config management logic. + +- :mod:`flake8.options.config` contains the logic for finding, parsing, and + merging configuration files. + +- :mod:`flake8.options.manager` contains the logic for managing customized + Flake8 command-line and configuration options. + +- :mod:`flake8.options.aggregator` uses objects from both of the above modules + to aggregate configuration into one object used by plugins and Flake8. + +""" diff --git a/contrib/flake8/options/aggregator.py b/contrib/flake8/options/aggregator.py new file mode 100644 index 0000000..5b8ab9c --- /dev/null +++ b/contrib/flake8/options/aggregator.py @@ -0,0 +1,78 @@ +"""Aggregation function for CLI specified options and config file options. + +This holds the logic that uses the collected and merged config files and +applies the user-specified command-line configuration on top of it. +""" +import logging + +from flake8.options import config + +LOG = logging.getLogger(__name__) + + +def aggregate_options(manager, config_finder, arglist=None, values=None): + """Aggregate and merge CLI and config file options. + + :param flake8.options.manager.OptionManager manager: + The instance of the OptionManager that we're presently using. + :param flake8.options.config.ConfigFileFinder config_finder: + The config file finder to use. + :param list arglist: + The list of arguments to pass to ``manager.parse_args``. In most cases + this will be None so ``parse_args`` uses ``sys.argv``. This is mostly + available to make testing easier. + :param optparse.Values values: + Previously parsed set of parsed options. + :returns: + Tuple of the parsed options and extra arguments returned by + ``manager.parse_args``. + :rtype: + tuple(optparse.Values, list) + """ + # Get defaults from the option parser + default_values, _ = manager.parse_args([], values=values) + # Get original CLI values so we can find additional config file paths and + # see if --config was specified. + original_values, _ = manager.parse_args(arglist) + + # Make our new configuration file mergerator + config_parser = config.MergedConfigParser( + option_manager=manager, + config_finder=config_finder, + ) + + # Get the parsed config + parsed_config = config_parser.parse(original_values.config, + original_values.isolated) + + # Extend the default ignore value with the extended default ignore list, + # registered by plugins. + extended_default_ignore = manager.extended_default_ignore.copy() + LOG.debug('Extended default ignore list: %s', + list(extended_default_ignore)) + extended_default_ignore.update(default_values.ignore) + default_values.ignore = list(extended_default_ignore) + LOG.debug('Merged default ignore list: %s', default_values.ignore) + + extended_default_select = manager.extended_default_select.copy() + LOG.debug('Extended default select list: %s', + list(extended_default_select)) + default_values.extended_default_select = extended_default_select + + # Merge values parsed from config onto the default values returned + for config_name, value in parsed_config.items(): + dest_name = config_name + # If the config name is somehow different from the destination name, + # fetch the destination name from our Option + if not hasattr(default_values, config_name): + dest_name = config_parser.config_options[config_name].dest + + LOG.debug('Overriding default value of (%s) for "%s" with (%s)', + getattr(default_values, dest_name, None), + dest_name, + value) + # Override the default values with the config values + setattr(default_values, dest_name, value) + + # Finally parse the command-line options + return manager.parse_args(arglist, default_values) diff --git a/contrib/flake8/options/config.py b/contrib/flake8/options/config.py new file mode 100644 index 0000000..71429af --- /dev/null +++ b/contrib/flake8/options/config.py @@ -0,0 +1,342 @@ +"""Config handling logic for Flake8.""" +import collections +import configparser +import logging +import os.path +import sys + +from flake8 import utils + +LOG = logging.getLogger(__name__) + +__all__ = ('ConfigFileFinder', 'MergedConfigParser') + + +class ConfigFileFinder(object): + """Encapsulate the logic for finding and reading config files.""" + + PROJECT_FILENAMES = ('setup.cfg', 'tox.ini') + + def __init__(self, program_name, args, extra_config_files): + """Initialize object to find config files. + + :param str program_name: + Name of the current program (e.g., flake8). + :param list args: + The extra arguments passed on the command-line. + :param list extra_config_files: + Extra configuration files specified by the user to read. + """ + # The values of --append-config from the CLI + extra_config_files = extra_config_files or [] + self.extra_config_files = [ + # Ensure the paths are absolute paths for local_config_files + os.path.abspath(f) for f in extra_config_files + ] + + # Platform specific settings + self.is_windows = sys.platform == 'win32' + self.xdg_home = os.environ.get('XDG_CONFIG_HOME', + os.path.expanduser('~/.config')) + + # Look for '.' files + self.program_config = '.' + program_name + self.program_name = program_name + + # List of filenames to find in the local/project directory + self.project_filenames = ('setup.cfg', 'tox.ini', self.program_config) + + self.local_directory = os.path.abspath(os.curdir) + + if not args: + args = ['.'] + self.parent = self.tail = os.path.abspath(os.path.commonprefix(args)) + + # caches to avoid double-reading config files + self._local_configs = None + self._user_config = None + self._cli_configs = {} + + @staticmethod + def _read_config(files): + config = configparser.RawConfigParser() + if isinstance(files, (str, type(u''))): + files = [files] + + found_files = [] + for filename in files: + try: + found_files.extend(config.read(filename)) + except UnicodeDecodeError: + LOG.exception("There was an error decoding a config file." + "The file with a problem was %s.", + filename) + except configparser.ParsingError: + LOG.exception("There was an error trying to parse a config " + "file. The file with a problem was %s.", + filename) + return (config, found_files) + + def cli_config(self, files): + """Read and parse the config file specified on the command-line.""" + if files not in self._cli_configs: + config, found_files = self._read_config(files) + if found_files: + LOG.debug('Found cli configuration files: %s', found_files) + self._cli_configs[files] = config + return self._cli_configs[files] + + def generate_possible_local_files(self): + """Find and generate all local config files.""" + tail = self.tail + parent = self.parent + found_config_files = False + while tail and not found_config_files: + for project_filename in self.project_filenames: + filename = os.path.abspath(os.path.join(parent, + project_filename)) + if os.path.exists(filename): + yield filename + found_config_files = True + self.local_directory = parent + (parent, tail) = os.path.split(parent) + + def local_config_files(self): + """Find all local config files which actually exist. + + Filter results from + :meth:`~ConfigFileFinder.generate_possible_local_files` based + on whether the filename exists or not. + + :returns: + List of files that exist that are local project config files with + extra config files appended to that list (which also exist). + :rtype: + [str] + """ + exists = os.path.exists + return [ + filename + for filename in self.generate_possible_local_files() + ] + [f for f in self.extra_config_files if exists(f)] + + def local_configs(self): + """Parse all local config files into one config object.""" + if self._local_configs is None: + config, found_files = self._read_config(self.local_config_files()) + if found_files: + LOG.debug('Found local configuration files: %s', found_files) + self._local_configs = config + return self._local_configs + + def user_config_file(self): + """Find the user-level config file.""" + if self.is_windows: + return os.path.expanduser('~\\' + self.program_config) + return os.path.join(self.xdg_home, self.program_name) + + def user_config(self): + """Parse the user config file into a config object.""" + if self._user_config is None: + config, found_files = self._read_config(self.user_config_file()) + if found_files: + LOG.debug('Found user configuration files: %s', found_files) + self._user_config = config + return self._user_config + + +class MergedConfigParser(object): + """Encapsulate merging different types of configuration files. + + This parses out the options registered that were specified in the + configuration files, handles extra configuration files, and returns + dictionaries with the parsed values. + """ + + #: Set of types that should use the + #: :meth:`~configparser.RawConfigParser.getint` method. + GETINT_TYPES = {'int', 'count'} + #: Set of actions that should use the + #: :meth:`~configparser.RawConfigParser.getbool` method. + GETBOOL_ACTIONS = {'store_true', 'store_false'} + + def __init__(self, option_manager, config_finder): + """Initialize the MergedConfigParser instance. + + :param flake8.options.manager.OptionManager option_manager: + Initialized OptionManager. + :param flake8.options.config.ConfigFileFinder config_finder: + Initialized ConfigFileFinder. + """ + #: Our instance of flake8.options.manager.OptionManager + self.option_manager = option_manager + #: The prog value for the cli parser + self.program_name = option_manager.program_name + #: Mapping of configuration option names to + #: :class:`~flake8.options.manager.Option` instances + self.config_options = option_manager.config_options_dict + #: Our instance of our :class:`~ConfigFileFinder` + self.config_finder = config_finder + + def _normalize_value(self, option, value): + final_value = option.normalize( + value, + self.config_finder.local_directory, + ) + LOG.debug('%r has been normalized to %r for option "%s"', + value, final_value, option.config_name) + return final_value + + def _parse_config(self, config_parser): + config_dict = {} + for option_name in config_parser.options(self.program_name): + if option_name not in self.config_options: + LOG.debug('Option "%s" is not registered. Ignoring.', + option_name) + continue + option = self.config_options[option_name] + + # Use the appropriate method to parse the config value + method = config_parser.get + if (option.type in self.GETINT_TYPES or + option.action in self.GETINT_TYPES): + method = config_parser.getint + elif option.action in self.GETBOOL_ACTIONS: + method = config_parser.getboolean + + value = method(self.program_name, option_name) + LOG.debug('Option "%s" returned value: %r', option_name, value) + + final_value = self._normalize_value(option, value) + config_dict[option.config_name] = final_value + + return config_dict + + def is_configured_by(self, config): + """Check if the specified config parser has an appropriate section.""" + return config.has_section(self.program_name) + + def parse_local_config(self): + """Parse and return the local configuration files.""" + config = self.config_finder.local_configs() + if not self.is_configured_by(config): + LOG.debug('Local configuration files have no %s section', + self.program_name) + return {} + + LOG.debug('Parsing local configuration files.') + return self._parse_config(config) + + def parse_user_config(self): + """Parse and return the user configuration files.""" + config = self.config_finder.user_config() + if not self.is_configured_by(config): + LOG.debug('User configuration files have no %s section', + self.program_name) + return {} + + LOG.debug('Parsing user configuration files.') + return self._parse_config(config) + + def parse_cli_config(self, config_path): + """Parse and return the file specified by --config.""" + config = self.config_finder.cli_config(config_path) + if not self.is_configured_by(config): + LOG.debug('CLI configuration files have no %s section', + self.program_name) + return {} + + LOG.debug('Parsing CLI configuration files.') + return self._parse_config(config) + + def merge_user_and_local_config(self): + """Merge the parsed user and local configuration files. + + :returns: + Dictionary of the parsed and merged configuration options. + :rtype: + dict + """ + user_config = self.parse_user_config() + config = self.parse_local_config() + + for option, value in user_config.items(): + config.setdefault(option, value) + + return config + + def parse(self, cli_config=None, isolated=False): + """Parse and return the local and user config files. + + First this copies over the parsed local configuration and then + iterates over the options in the user configuration and sets them if + they were not set by the local configuration file. + + :param str cli_config: + Value of --config when specified at the command-line. Overrides + all other config files. + :param bool isolated: + Determines if we should parse configuration files at all or not. + If running in isolated mode, we ignore all configuration files + :returns: + Dictionary of parsed configuration options + :rtype: + dict + """ + if isolated: + LOG.debug('Refusing to parse configuration files due to user-' + 'requested isolation') + return {} + + if cli_config: + LOG.debug('Ignoring user and locally found configuration files. ' + 'Reading only configuration from "%s" specified via ' + '--config by the user', cli_config) + return self.parse_cli_config(cli_config) + + return self.merge_user_and_local_config() + + +def get_local_plugins(config_finder, cli_config=None, isolated=False): + """Get local plugins lists from config files. + + :param flake8.options.config.ConfigFileFinder config_finder: + The config file finder to use. + :param str cli_config: + Value of --config when specified at the command-line. Overrides + all other config files. + :param bool isolated: + Determines if we should parse configuration files at all or not. + If running in isolated mode, we ignore all configuration files + :returns: + LocalPlugins namedtuple containing two lists of plugin strings, + one for extension (checker) plugins and one for report plugins. + :rtype: + flake8.options.config.LocalPlugins + """ + local_plugins = LocalPlugins(extension=[], report=[]) + if isolated: + LOG.debug('Refusing to look for local plugins in configuration' + 'files due to user-requested isolation') + return local_plugins + + if cli_config: + LOG.debug('Reading local plugins only from "%s" specified via ' + '--config by the user', cli_config) + config = config_finder.cli_config(cli_config) + else: + config = config_finder.local_configs() + + section = '%s:local-plugins' % config_finder.program_name + for plugin_type in ['extension', 'report']: + if config.has_option(section, plugin_type): + local_plugins_string = config.get(section, plugin_type).strip() + plugin_type_list = getattr(local_plugins, plugin_type) + plugin_type_list.extend(utils.parse_comma_separated_list( + local_plugins_string, + regexp=utils.LOCAL_PLUGIN_LIST_RE, + )) + return local_plugins + + +LocalPlugins = collections.namedtuple('LocalPlugins', 'extension report') diff --git a/contrib/flake8/options/manager.py b/contrib/flake8/options/manager.py new file mode 100644 index 0000000..5b4796f --- /dev/null +++ b/contrib/flake8/options/manager.py @@ -0,0 +1,325 @@ +"""Option handling and Option management logic.""" +import collections +import logging +import optparse # pylint: disable=deprecated-module + +from flake8 import utils + +LOG = logging.getLogger(__name__) + + +class Option(object): + """Our wrapper around an optparse.Option object to add features.""" + + def __init__(self, short_option_name=None, long_option_name=None, + # Options below here are taken from the optparse.Option class + action=None, default=None, type=None, dest=None, + nargs=None, const=None, choices=None, callback=None, + callback_args=None, callback_kwargs=None, help=None, + metavar=None, + # Options below here are specific to Flake8 + parse_from_config=False, comma_separated_list=False, + normalize_paths=False): + """Initialize an Option instance wrapping optparse.Option. + + The following are all passed directly through to optparse. + + :param str short_option_name: + The short name of the option (e.g., ``-x``). This will be the + first argument passed to :class:`~optparse.Option`. + :param str long_option_name: + The long name of the option (e.g., ``--xtra-long-option``). This + will be the second argument passed to :class:`~optparse.Option`. + :param str action: + Any action allowed by :mod:`optparse`. + :param default: + Default value of the option. + :param type: + Any type allowed by :mod:`optparse`. + :param dest: + Attribute name to store parsed option value as. + :param nargs: + Number of arguments to parse for this option. + :param const: + Constant value to store on a common destination. Usually used in + conjuntion with ``action="store_const"``. + :param iterable choices: + Possible values for the option. + :param callable callback: + Callback used if the action is ``"callback"``. + :param iterable callback_args: + Additional positional arguments to the callback callable. + :param dictionary callback_kwargs: + Keyword arguments to the callback callable. + :param str help: + Help text displayed in the usage information. + :param str metavar: + Name to use instead of the long option name for help text. + + The following parameters are for Flake8's option handling alone. + + :param bool parse_from_config: + Whether or not this option should be parsed out of config files. + :param bool comma_separated_list: + Whether the option is a comma separated list when parsing from a + config file. + :param bool normalize_paths: + Whether the option is expecting a path or list of paths and should + attempt to normalize the paths to absolute paths. + """ + self.short_option_name = short_option_name + self.long_option_name = long_option_name + self.option_args = [ + x for x in (short_option_name, long_option_name) if x is not None + ] + self.option_kwargs = { + 'action': action, + 'default': default, + 'type': type, + 'dest': self._make_dest(dest), + 'nargs': nargs, + 'const': const, + 'choices': choices, + 'callback': callback, + 'callback_args': callback_args, + 'callback_kwargs': callback_kwargs, + 'help': help, + 'metavar': metavar, + } + # Set attributes for our option arguments + for key, value in self.option_kwargs.items(): + setattr(self, key, value) + + # Set our custom attributes + self.parse_from_config = parse_from_config + self.comma_separated_list = comma_separated_list + self.normalize_paths = normalize_paths + + self.config_name = None + if parse_from_config: + if not long_option_name: + raise ValueError('When specifying parse_from_config=True, ' + 'a long_option_name must also be specified.') + self.config_name = long_option_name[2:].replace('-', '_') + + self._opt = None + + def __repr__(self): # noqa: D105 + return ( + 'Option({0}, {1}, action={action}, default={default}, ' + 'dest={dest}, type={type}, callback={callback}, help={help},' + ' callback={callback}, callback_args={callback_args}, ' + 'callback_kwargs={callback_kwargs}, metavar={metavar})' + ).format(self.short_option_name, self.long_option_name, + **self.option_kwargs) + + def _make_dest(self, dest): + if dest: + return dest + + if self.long_option_name: + return self.long_option_name[2:].replace('-', '_') + return self.short_option_name[1] + + def normalize(self, value, *normalize_args): + """Normalize the value based on the option configuration.""" + if self.normalize_paths: + # Decide whether to parse a list of paths or a single path + normalize = utils.normalize_path + if self.comma_separated_list: + normalize = utils.normalize_paths + return normalize(value, *normalize_args) + elif self.comma_separated_list: + return utils.parse_comma_separated_list(value) + return value + + def normalize_from_setuptools(self, value): + """Normalize the value received from setuptools.""" + value = self.normalize(value) + if self.type == 'int' or self.action == 'count': + return int(value) + if self.action in ('store_true', 'store_false'): + value = str(value).upper() + if value in ('1', 'T', 'TRUE', 'ON'): + return True + if value in ('0', 'F', 'FALSE', 'OFF'): + return False + return value + + def to_optparse(self): + """Convert a Flake8 Option to an optparse Option.""" + if self._opt is None: + self._opt = optparse.Option(*self.option_args, + **self.option_kwargs) + return self._opt + + +PluginVersion = collections.namedtuple("PluginVersion", + ["name", "version", "local"]) + + +class OptionManager(object): + """Manage Options and OptionParser while adding post-processing.""" + + def __init__(self, prog=None, version=None, + usage='%prog [options] file file ...'): + """Initialize an instance of an OptionManager. + + :param str prog: + Name of the actual program (e.g., flake8). + :param str version: + Version string for the program. + :param str usage: + Basic usage string used by the OptionParser. + """ + self.parser = optparse.OptionParser(prog=prog, version=version, + usage=usage) + self.config_options_dict = {} + self.options = [] + self.program_name = prog + self.version = version + self.registered_plugins = set() + self.extended_default_ignore = set() + self.extended_default_select = set() + + @staticmethod + def format_plugin(plugin): + """Convert a PluginVersion into a dictionary mapping name to value.""" + return {attr: getattr(plugin, attr) for attr in ["name", "version"]} + + def add_option(self, *args, **kwargs): + """Create and register a new option. + + See parameters for :class:`~flake8.options.manager.Option` for + acceptable arguments to this method. + + .. note:: + + ``short_option_name`` and ``long_option_name`` may be specified + positionally as they are with optparse normally. + """ + if len(args) == 1 and args[0].startswith('--'): + args = (None, args[0]) + option = Option(*args, **kwargs) + self.parser.add_option(option.to_optparse()) + self.options.append(option) + if option.parse_from_config: + name = option.config_name + self.config_options_dict[name] = option + self.config_options_dict[name.replace('_', '-')] = option + LOG.debug('Registered option "%s".', option) + + def remove_from_default_ignore(self, error_codes): + """Remove specified error codes from the default ignore list. + + :param list error_codes: + List of strings that are the error/warning codes to attempt to + remove from the extended default ignore list. + """ + LOG.debug('Removing %r from the default ignore list', error_codes) + for error_code in error_codes: + try: + self.extended_default_ignore.remove(error_code) + except (ValueError, KeyError): + LOG.debug('Attempted to remove %s from default ignore' + ' but it was not a member of the list.', error_code) + + def extend_default_ignore(self, error_codes): + """Extend the default ignore list with the error codes provided. + + :param list error_codes: + List of strings that are the error/warning codes with which to + extend the default ignore list. + """ + LOG.debug('Extending default ignore list with %r', error_codes) + self.extended_default_ignore.update(error_codes) + + def extend_default_select(self, error_codes): + """Extend the default select list with the error codes provided. + + :param list error_codes: + List of strings that are the error/warning codes with which + to extend the default select list. + """ + LOG.debug('Extending default select list with %r', error_codes) + self.extended_default_select.update(error_codes) + + def generate_versions(self, format_str='%(name)s: %(version)s', + join_on=', '): + """Generate a comma-separated list of versions of plugins.""" + return join_on.join( + format_str % self.format_plugin(plugin) + for plugin in sorted(self.registered_plugins) + ) + + def update_version_string(self): + """Update the flake8 version string.""" + self.parser.version = ( + self.version + ' (' + self.generate_versions() + ') ' + + utils.get_python_version() + ) + + def generate_epilog(self): + """Create an epilog with the version and name of each of plugin.""" + plugin_version_format = '%(name)s: %(version)s' + self.parser.epilog = 'Installed plugins: ' + self.generate_versions( + plugin_version_format + ) + + def _normalize(self, options): + for option in self.options: + old_value = getattr(options, option.dest) + setattr(options, option.dest, option.normalize(old_value)) + + def parse_args(self, args=None, values=None): + """Proxy to calling the OptionParser's parse_args method.""" + self.generate_epilog() + self.update_version_string() + options, xargs = self.parser.parse_args(args, values) + self._normalize(options) + return options, xargs + + def parse_known_args(self, args=None, values=None): + """Parse only the known arguments from the argument values. + + Replicate a little argparse behaviour while we're still on + optparse. + """ + self.generate_epilog() + self.update_version_string() + # Taken from optparse.OptionParser.parse_args + rargs = self.parser._get_args(args) + if values is None: + values = self.parser.get_default_values() + + self.parser.rargs = rargs + self.parser.largs = largs = [] + self.parser.values = values + + while rargs: + # NOTE(sigmavirus24): If we only care about *known* options, then + # we should just shift the bad option over to the largs list and + # carry on. + # Unfortunately, we need to rely on a private method here. + try: + self.parser._process_args(largs, rargs, values) + except (optparse.BadOptionError, optparse.OptionValueError) as err: + self.parser.largs.append(err.opt_str) + + args = largs + rargs + options, xargs = self.parser.check_values(values, args) + self._normalize(options) + return options, xargs + + def register_plugin(self, name, version, local=False): + """Register a plugin relying on the OptionManager. + + :param str name: + The name of the checker itself. This will be the ``name`` + attribute of the class or function loaded from the entry-point. + :param str version: + The version of the checker that we're using. + :param bool local: + Whether the plugin is local to the project/repository or not. + """ + self.registered_plugins.add(PluginVersion(name, version, local)) diff --git a/contrib/flake8/plugins/__init__.py b/contrib/flake8/plugins/__init__.py new file mode 100644 index 0000000..fda6a44 --- /dev/null +++ b/contrib/flake8/plugins/__init__.py @@ -0,0 +1 @@ +"""Submodule of built-in plugins and plugin managers.""" diff --git a/contrib/flake8/plugins/_trie.py b/contrib/flake8/plugins/_trie.py new file mode 100644 index 0000000..17c226f --- /dev/null +++ b/contrib/flake8/plugins/_trie.py @@ -0,0 +1,97 @@ +"""Independent implementation of a Trie tree.""" + +__all__ = ('Trie', 'TrieNode') + + +def _iterate_stringlike_objects(string): + for i in range(len(string)): + yield string[i:i + 1] + + +class Trie(object): + """The object that manages the trie nodes.""" + + def __init__(self): + """Initialize an empty trie.""" + self.root = TrieNode(None, None) + + def add(self, path, node_data): + """Add the node data to the path described.""" + node = self.root + for prefix in _iterate_stringlike_objects(path): + child = node.find_prefix(prefix) + if child is None: + child = node.add_child(prefix, []) + node = child + node.data.append(node_data) + + def find(self, path): + """Find a node based on the path provided.""" + node = self.root + for prefix in _iterate_stringlike_objects(path): + child = node.find_prefix(prefix) + if child is None: + return None + node = child + return node + + def traverse(self): + """Traverse this tree. + + This performs a depth-first pre-order traversal of children in this + tree. It returns the results consistently by first sorting the + children based on their prefix and then traversing them in + alphabetical order. + """ + return self.root.traverse() + + +class TrieNode(object): + """The majority of the implementation details of a Trie.""" + + def __init__(self, prefix, data, children=None): + """Initialize a TrieNode with data and children.""" + self.children = children or {} + self.data = data + self.prefix = prefix + + def __repr__(self): + """Generate an easy to read representation of the node.""" + return 'TrieNode(prefix={0}, data={1})'.format( + self.prefix, self.data + ) + + def find_prefix(self, prefix): + """Find the prefix in the children of this node. + + :returns: A child matching the prefix or None. + :rtype: :class:`~TrieNode` or None + """ + return self.children.get(prefix, None) + + def add_child(self, prefix, data, children=None): + """Create and add a new child node. + + :returns: The newly created node + :rtype: :class:`~TrieNode` + """ + new_node = TrieNode(prefix, data, children) + self.children[prefix] = new_node + return new_node + + def traverse(self): + """Traverse children of this node. + + This performs a depth-first pre-order traversal of the remaining + children in this sub-tree. It returns the results consistently by + first sorting the children based on their prefix and then traversing + them in alphabetical order. + """ + if not self.children: + return + + for prefix in sorted(self.children): + child = self.children[prefix] + yield child + for child in child.traverse(): + yield child diff --git a/contrib/flake8/plugins/manager.py b/contrib/flake8/plugins/manager.py new file mode 100644 index 0000000..503dfbb --- /dev/null +++ b/contrib/flake8/plugins/manager.py @@ -0,0 +1,558 @@ +"""Plugin loading and management logic and classes.""" +import collections +import logging + +import pkg_resources + +from flake8 import exceptions +from flake8 import utils +from flake8.plugins import notifier + +LOG = logging.getLogger(__name__) + +__all__ = ( + 'Checkers', + 'Listeners', + 'Plugin', + 'PluginManager', + 'ReportFormatters', +) + +NO_GROUP_FOUND = object() + + +class Plugin(object): + """Wrap an EntryPoint from setuptools and other logic.""" + + def __init__(self, name, entry_point, local=False): + """Initialize our Plugin. + + :param str name: + Name of the entry-point as it was registered with setuptools. + :param entry_point: + EntryPoint returned by setuptools. + :type entry_point: + setuptools.EntryPoint + :param bool local: + Is this a repo-local plugin? + """ + self.name = name + self.entry_point = entry_point + self.local = local + self._plugin = None + self._parameters = None + self._parameter_names = None + self._group = None + self._plugin_name = None + self._version = None + + def __repr__(self): + """Provide an easy to read description of the current plugin.""" + return 'Plugin(name="{0}", entry_point="{1}")'.format( + self.name, self.entry_point + ) + + def to_dictionary(self): + """Convert this plugin to a dictionary.""" + return { + 'name': self.name, + 'parameters': self.parameters, + 'parameter_names': self.parameter_names, + 'plugin': self.plugin, + 'plugin_name': self.plugin_name, + } + + def is_in_a_group(self): + """Determine if this plugin is in a group. + + :returns: + True if the plugin is in a group, otherwise False. + :rtype: + bool + """ + return self.group() is not None + + def group(self): + """Find and parse the group the plugin is in.""" + if self._group is None: + name = self.name.split('.', 1) + if len(name) > 1: + self._group = name[0] + else: + self._group = NO_GROUP_FOUND + if self._group is NO_GROUP_FOUND: + return None + return self._group + + @property + def parameters(self): + """List of arguments that need to be passed to the plugin.""" + if self._parameters is None: + self._parameters = utils.parameters_for(self) + return self._parameters + + @property + def parameter_names(self): + """List of argument names that need to be passed to the plugin.""" + if self._parameter_names is None: + self._parameter_names = list(self.parameters) + return self._parameter_names + + @property + def plugin(self): + """Load and return the plugin associated with the entry-point. + + This property implicitly loads the plugin and then caches it. + """ + self.load_plugin() + return self._plugin + + @property + def version(self): + """Return the version of the plugin.""" + if self._version is None: + if self.is_in_a_group(): + self._version = version_for(self) + else: + self._version = self.plugin.version + + return self._version + + @property + def plugin_name(self): + """Return the name of the plugin.""" + if self._plugin_name is None: + if self.is_in_a_group(): + self._plugin_name = self.group() + else: + self._plugin_name = self.plugin.name + + return self._plugin_name + + @property + def off_by_default(self): + """Return whether the plugin is ignored by default.""" + return getattr(self.plugin, 'off_by_default', False) + + def execute(self, *args, **kwargs): + r"""Call the plugin with \*args and \*\*kwargs.""" + return self.plugin(*args, **kwargs) # pylint: disable=not-callable + + def _load(self, verify_requirements): + # Avoid relying on hasattr() here. + resolve = getattr(self.entry_point, 'resolve', None) + require = getattr(self.entry_point, 'require', None) + if resolve and require: + if verify_requirements: + LOG.debug('Verifying plugin "%s"\'s requirements.', + self.name) + require() + self._plugin = resolve() + else: + self._plugin = self.entry_point.load( + require=verify_requirements + ) + if not callable(self._plugin): + msg = ('Plugin %r is not a callable. It might be written for an' + ' older version of flake8 and might not work with this' + ' version' % self._plugin) + LOG.critical(msg) + raise TypeError(msg) + + def load_plugin(self, verify_requirements=False): + """Retrieve the plugin for this entry-point. + + This loads the plugin, stores it on the instance and then returns it. + It does not reload it after the first time, it merely returns the + cached plugin. + + :param bool verify_requirements: + Whether or not to make setuptools verify that the requirements for + the plugin are satisfied. + :returns: + Nothing + """ + if self._plugin is None: + LOG.info('Loading plugin "%s" from entry-point.', self.name) + try: + self._load(verify_requirements) + except Exception as load_exception: + LOG.exception(load_exception) + failed_to_load = exceptions.FailedToLoadPlugin( + plugin=self, + exception=load_exception, + ) + LOG.critical(str(failed_to_load)) + raise failed_to_load + + def enable(self, optmanager, options=None): + """Remove plugin name from the default ignore list.""" + optmanager.remove_from_default_ignore([self.name]) + optmanager.extend_default_select([self.name]) + if not options: + return + try: + options.ignore.remove(self.name) + except (ValueError, KeyError): + LOG.debug('Attempted to remove %s from the ignore list but it was ' + 'not a member of the list.', self.name) + + def disable(self, optmanager): + """Add the plugin name to the default ignore list.""" + optmanager.extend_default_ignore([self.name]) + + def provide_options(self, optmanager, options, extra_args): + """Pass the parsed options and extra arguments to the plugin.""" + parse_options = getattr(self.plugin, 'parse_options', None) + if parse_options is not None: + LOG.debug('Providing options to plugin "%s".', self.name) + try: + parse_options(optmanager, options, extra_args) + except TypeError: + parse_options(options) + + if self.name in options.enable_extensions: + self.enable(optmanager, options) + + def register_options(self, optmanager): + """Register the plugin's command-line options on the OptionManager. + + :param optmanager: + Instantiated OptionManager to register options on. + :type optmanager: + flake8.options.manager.OptionManager + :returns: + Nothing + """ + add_options = getattr(self.plugin, 'add_options', None) + if add_options is not None: + LOG.debug( + 'Registering options from plugin "%s" on OptionManager %r', + self.name, optmanager + ) + add_options(optmanager) + + if self.off_by_default: + self.disable(optmanager) + + +class PluginManager(object): # pylint: disable=too-few-public-methods + """Find and manage plugins consistently.""" + + def __init__(self, namespace, + verify_requirements=False, local_plugins=None): + """Initialize the manager. + + :param str namespace: + Namespace of the plugins to manage, e.g., 'flake8.extension'. + :param list local_plugins: + Plugins from config (as "X = path.to:Plugin" strings). + :param bool verify_requirements: + Whether or not to make setuptools verify that the requirements for + the plugin are satisfied. + """ + self.namespace = namespace + self.verify_requirements = verify_requirements + self.plugins = {} + self.names = [] + self._load_local_plugins(local_plugins or []) + self._load_entrypoint_plugins() + + def _load_local_plugins(self, local_plugins): + """Load local plugins from config. + + :param list local_plugins: + Plugins from config (as "X = path.to:Plugin" strings). + """ + for plugin_str in local_plugins: + entry_point = pkg_resources.EntryPoint.parse(plugin_str) + self._load_plugin_from_entrypoint(entry_point, local=True) + + def _load_entrypoint_plugins(self): + LOG.info('Loading entry-points for "%s".', self.namespace) + for entry_point in pkg_resources.iter_entry_points(self.namespace): + self._load_plugin_from_entrypoint(entry_point) + + def _load_plugin_from_entrypoint(self, entry_point, local=False): + """Load a plugin from a setuptools EntryPoint. + + :param EntryPoint entry_point: + EntryPoint to load plugin from. + :param bool local: + Is this a repo-local plugin? + """ + name = entry_point.name + self.plugins[name] = Plugin(name, entry_point, local=local) + self.names.append(name) + LOG.debug('Loaded %r for plugin "%s".', self.plugins[name], name) + + def map(self, func, *args, **kwargs): + r"""Call ``func`` with the plugin and \*args and \**kwargs after. + + This yields the return value from ``func`` for each plugin. + + :param collections.Callable func: + Function to call with each plugin. Signature should at least be: + + .. code-block:: python + + def myfunc(plugin): + pass + + Any extra positional or keyword arguments specified with map will + be passed along to this function after the plugin. The plugin + passed is a :class:`~flake8.plugins.manager.Plugin`. + :param args: + Positional arguments to pass to ``func`` after each plugin. + :param kwargs: + Keyword arguments to pass to ``func`` after each plugin. + """ + for name in self.names: + yield func(self.plugins[name], *args, **kwargs) + + def versions(self): + # () -> (str, str) + """Generate the versions of plugins. + + :returns: + Tuples of the plugin_name and version + :rtype: + tuple + """ + plugins_seen = set() + for entry_point_name in self.names: + plugin = self.plugins[entry_point_name] + plugin_name = plugin.plugin_name + if plugin.plugin_name in plugins_seen: + continue + plugins_seen.add(plugin_name) + yield (plugin_name, plugin.version) + + +def version_for(plugin): + # (Plugin) -> Union[str, NoneType] + """Determine the version of a plugin by it's module. + + :param plugin: + The loaded plugin + :type plugin: + Plugin + :returns: + version string for the module + :rtype: + str + """ + module_name = plugin.plugin.__module__ + try: + module = __import__(module_name) + except ImportError: + return None + + return getattr(module, '__version__', None) + + +class PluginTypeManager(object): + """Parent class for most of the specific plugin types.""" + + namespace = None + + def __init__(self, local_plugins=None): + """Initialize the plugin type's manager. + + :param list local_plugins: + Plugins from config file instead of entry-points + """ + self.manager = PluginManager( + self.namespace, local_plugins=local_plugins) + self.plugins_loaded = False + + def __contains__(self, name): + """Check if the entry-point name is in this plugin type manager.""" + LOG.debug('Checking for "%s" in plugin type manager.', name) + return name in self.plugins + + def __getitem__(self, name): + """Retrieve a plugin by its name.""" + LOG.debug('Retrieving plugin for "%s".', name) + return self.plugins[name] + + def get(self, name, default=None): + """Retrieve the plugin referred to by ``name`` or return the default. + + :param str name: + Name of the plugin to retrieve. + :param default: + Default value to return. + :returns: + Plugin object referred to by name, if it exists. + :rtype: + :class:`Plugin` + """ + if name in self: + return self[name] + return default + + @property + def names(self): + """Proxy attribute to underlying manager.""" + return self.manager.names + + @property + def plugins(self): + """Proxy attribute to underlying manager.""" + return self.manager.plugins + + @staticmethod + def _generate_call_function(method_name, optmanager, *args, **kwargs): + def generated_function(plugin): # noqa: D105 + method = getattr(plugin, method_name, None) + if (method is not None and + isinstance(method, collections.Callable)): + return method(optmanager, *args, **kwargs) + return generated_function + + def load_plugins(self): + """Load all plugins of this type that are managed by this manager.""" + if self.plugins_loaded: + return + + def load_plugin(plugin): + """Call each plugin's load_plugin method.""" + return plugin.load_plugin() + + plugins = list(self.manager.map(load_plugin)) + # Do not set plugins_loaded if we run into an exception + self.plugins_loaded = True + return plugins + + def register_plugin_versions(self, optmanager): + """Register the plugins and their versions with the OptionManager.""" + self.load_plugins() + for (plugin_name, version) in self.manager.versions(): + optmanager.register_plugin(name=plugin_name, version=version) + + def register_options(self, optmanager): + """Register all of the checkers' options to the OptionManager.""" + self.load_plugins() + call_register_options = self._generate_call_function( + 'register_options', optmanager, + ) + + list(self.manager.map(call_register_options)) + + def provide_options(self, optmanager, options, extra_args): + """Provide parsed options and extra arguments to the plugins.""" + call_provide_options = self._generate_call_function( + 'provide_options', optmanager, options, extra_args, + ) + + list(self.manager.map(call_provide_options)) + + +class NotifierBuilderMixin(object): # pylint: disable=too-few-public-methods + """Mixin class that builds a Notifier from a PluginManager.""" + + def build_notifier(self): + """Build a Notifier for our Listeners. + + :returns: + Object to notify our listeners of certain error codes and + warnings. + :rtype: + :class:`~flake8.notifier.Notifier` + """ + notifier_trie = notifier.Notifier() + for name in self.names: + notifier_trie.register_listener(name, self.manager[name]) + return notifier_trie + + +class Checkers(PluginTypeManager): + """All of the checkers registered through entry-points or config.""" + + namespace = 'flake8.extension' + + def checks_expecting(self, argument_name): + """Retrieve checks that expect an argument with the specified name. + + Find all checker plugins that are expecting a specific argument. + """ + for plugin in self.plugins.values(): + if argument_name == plugin.parameter_names[0]: + yield plugin + + def to_dictionary(self): + """Return a dictionary of AST and line-based plugins.""" + return { + 'ast_plugins': [ + plugin.to_dictionary() for plugin in self.ast_plugins + ], + 'logical_line_plugins': [ + plugin.to_dictionary() for plugin in self.logical_line_plugins + ], + 'physical_line_plugins': [ + plugin.to_dictionary() for plugin in self.physical_line_plugins + ], + } + + def register_options(self, optmanager): + """Register all of the checkers' options to the OptionManager. + + This also ensures that plugins that are not part of a group and are + enabled by default are enabled on the option manager. + """ + # NOTE(sigmavirus24) We reproduce a little of + # PluginTypeManager.register_options to reduce the number of times + # that we loop over the list of plugins. Instead of looping twice, + # option registration and enabling the plugin, we loop once with one + # function to map over the plugins. + self.load_plugins() + call_register_options = self._generate_call_function( + 'register_options', optmanager, + ) + + def register_and_enable(plugin): + call_register_options(plugin) + if plugin.group() is None and not plugin.off_by_default: + plugin.enable(optmanager) + + list(self.manager.map(register_and_enable)) + + @property + def ast_plugins(self): + """List of plugins that expect the AST tree.""" + plugins = getattr(self, '_ast_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('tree')) + self._ast_plugins = plugins + return plugins + + @property + def logical_line_plugins(self): + """List of plugins that expect the logical lines.""" + plugins = getattr(self, '_logical_line_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('logical_line')) + self._logical_line_plugins = plugins + return plugins + + @property + def physical_line_plugins(self): + """List of plugins that expect the physical lines.""" + plugins = getattr(self, '_physical_line_plugins', []) + if not plugins: + plugins = list(self.checks_expecting('physical_line')) + self._physical_line_plugins = plugins + return plugins + + +class Listeners(PluginTypeManager, NotifierBuilderMixin): + """All of the listeners registered through entry-points or config.""" + + namespace = 'flake8.listen' + + +class ReportFormatters(PluginTypeManager): + """All of the report formatters registered through entry-points/config.""" + + namespace = 'flake8.report' diff --git a/contrib/flake8/plugins/notifier.py b/contrib/flake8/plugins/notifier.py new file mode 100644 index 0000000..dc255c4 --- /dev/null +++ b/contrib/flake8/plugins/notifier.py @@ -0,0 +1,46 @@ +"""Implementation of the class that registers and notifies listeners.""" +from flake8.plugins import _trie + + +class Notifier(object): + """Object that tracks and notifies listener objects.""" + + def __init__(self): + """Initialize an empty notifier object.""" + self.listeners = _trie.Trie() + + def listeners_for(self, error_code): + """Retrieve listeners for an error_code. + + There may be listeners registered for E1, E100, E101, E110, E112, and + E126. To get all the listeners for one of E100, E101, E110, E112, or + E126 you would also need to incorporate the listeners for E1 (since + they're all in the same class). + + Example usage: + + .. code-block:: python + + from flake8 import notifier + + n = notifier.Notifier() + # register listeners + for listener in n.listeners_for('W102'): + listener.notify(...) + """ + path = error_code + while path: + node = self.listeners.find(path) + listeners = getattr(node, 'data', []) + for listener in listeners: + yield listener + path = path[:-1] + + def notify(self, error_code, *args, **kwargs): + """Notify all listeners for the specified error code.""" + for listener in self.listeners_for(error_code): + listener.notify(error_code, *args, **kwargs) + + def register_listener(self, error_code, listener): + """Register a listener for a specific error_code.""" + self.listeners.add(error_code, listener) diff --git a/contrib/flake8/plugins/pyflakes.py b/contrib/flake8/plugins/pyflakes.py new file mode 100644 index 0000000..bc19291 --- /dev/null +++ b/contrib/flake8/plugins/pyflakes.py @@ -0,0 +1,163 @@ +"""Plugin built-in to Flake8 to treat pyflakes as a plugin.""" +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +try: + # The 'demandimport' breaks pyflakes and flake8.plugins.pyflakes + from mercurial import demandimport +except ImportError: + pass +else: + demandimport.disable() +import os + +import pyflakes +import pyflakes.checker + +from flake8 import utils + + +FLAKE8_PYFLAKES_CODES = { + 'UnusedImport': 'F401', + 'ImportShadowedByLoopVar': 'F402', + 'ImportStarUsed': 'F403', + 'LateFutureImport': 'F404', + 'ImportStarUsage': 'F405', + 'ImportStarNotPermitted': 'F406', + 'FutureFeatureNotDefined': 'F407', + 'MultiValueRepeatedKeyLiteral': 'F601', + 'MultiValueRepeatedKeyVariable': 'F602', + 'TooManyExpressionsInStarredAssignment': 'F621', + 'TwoStarredExpressions': 'F622', + 'AssertTuple': 'F631', + 'BreakOutsideLoop': 'F701', + 'ContinueOutsideLoop': 'F702', + 'ContinueInFinally': 'F703', + 'YieldOutsideFunction': 'F704', + 'ReturnWithArgsInsideGenerator': 'F705', + 'ReturnOutsideFunction': 'F706', + 'DefaultExceptNotLast': 'F707', + 'DoctestSyntaxError': 'F721', + 'RedefinedWhileUnused': 'F811', + 'RedefinedInListComp': 'F812', + 'UndefinedName': 'F821', + 'UndefinedExport': 'F822', + 'UndefinedLocal': 'F823', + 'DuplicateArgument': 'F831', + 'UnusedVariable': 'F841', +} + + +def patch_pyflakes(): + """Add error codes to Pyflakes messages.""" + for name, obj in vars(pyflakes.messages).items(): + if name[0].isupper() and obj.message: + obj.flake8_msg = '%s %s' % ( + FLAKE8_PYFLAKES_CODES.get(name, 'F999'), obj.message + ) + + +patch_pyflakes() + + +class FlakesChecker(pyflakes.checker.Checker): + """Subclass the Pyflakes checker to conform with the flake8 API.""" + + name = 'pyflakes' + version = pyflakes.__version__ + with_doctest = False + include_in_doctest = [] + exclude_from_doctest = [] + + def __init__(self, tree, filename): + """Initialize the PyFlakes plugin with an AST tree and filename.""" + filename = utils.normalize_paths(filename)[0] + with_doctest = self.with_doctest + included_by = [include for include in self.include_in_doctest + if include != '' and filename.startswith(include)] + if included_by: + with_doctest = True + + for exclude in self.exclude_from_doctest: + if exclude != '' and filename.startswith(exclude): + with_doctest = False + overlaped_by = [include for include in included_by + if include.startswith(exclude)] + + if overlaped_by: + with_doctest = True + + super(FlakesChecker, self).__init__(tree, filename, + withDoctest=with_doctest) + + @classmethod + def add_options(cls, parser): + """Register options for PyFlakes on the Flake8 OptionManager.""" + parser.add_option( + '--builtins', parse_from_config=True, comma_separated_list=True, + help="define more built-ins, comma separated", + ) + parser.add_option( + '--doctests', default=False, action='store_true', + parse_from_config=True, + help="check syntax of the doctests", + ) + parser.add_option( + '--include-in-doctest', default='', + dest='include_in_doctest', parse_from_config=True, + comma_separated_list=True, normalize_paths=True, + help='Run doctests only on these files', + type='string', + ) + parser.add_option( + '--exclude-from-doctest', default='', + dest='exclude_from_doctest', parse_from_config=True, + comma_separated_list=True, normalize_paths=True, + help='Skip these files when running doctests', + type='string', + ) + + @classmethod + def parse_options(cls, options): + """Parse option values from Flake8's OptionManager.""" + if options.builtins: + cls.builtIns = cls.builtIns.union(options.builtins) + cls.with_doctest = options.doctests + + included_files = [] + for included_file in options.include_in_doctest: + if included_file == '': + continue + if not included_file.startswith((os.sep, './', '~/')): + included_files.append('./' + included_file) + else: + included_files.append(included_file) + cls.include_in_doctest = utils.normalize_paths(included_files) + + excluded_files = [] + for excluded_file in options.exclude_from_doctest: + if excluded_file == '': + continue + if not excluded_file.startswith((os.sep, './', '~/')): + excluded_files.append('./' + excluded_file) + else: + excluded_files.append(excluded_file) + cls.exclude_from_doctest = utils.normalize_paths(excluded_files) + + inc_exc = set(cls.include_in_doctest).intersection( + cls.exclude_from_doctest + ) + if inc_exc: + raise ValueError('"%s" was specified in both the ' + 'include-in-doctest and exclude-from-doctest ' + 'options. You are not allowed to specify it in ' + 'both for doctesting.' % inc_exc) + + def run(self): + """Run the plugin.""" + for message in self.messages: + col = getattr(message, 'col', 0) + yield (message.lineno, + col, + (message.flake8_msg % message.message_args), + message.__class__) diff --git a/contrib/flake8/processor.py b/contrib/flake8/processor.py new file mode 100644 index 0000000..3827a26 --- /dev/null +++ b/contrib/flake8/processor.py @@ -0,0 +1,465 @@ +"""Module containing our file processor that tokenizes a file for checks.""" +import contextlib +import io +import logging +import sys +import tokenize + +import flake8 +from flake8 import defaults +from flake8 import exceptions +from flake8 import utils + +LOG = logging.getLogger(__name__) +PyCF_ONLY_AST = 1024 +NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) +# Work around Python < 2.6 behaviour, which does not generate NL after +# a comment which is on a line by itself. +COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' + +SKIP_TOKENS = frozenset([tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, + tokenize.DEDENT]) + + +class FileProcessor(object): + """Processes a file and holdes state. + + This processes a file by generating tokens, logical and physical lines, + and AST trees. This also provides a way of passing state about the file + to checks expecting that state. Any public attribute on this object can + be requested by a plugin. The known public attributes are: + + - :attr:`blank_before` + - :attr:`blank_lines` + - :attr:`checker_state` + - :attr:`indent_char` + - :attr:`indent_level` + - :attr:`line_number` + - :attr:`logical_line` + - :attr:`max_line_length` + - :attr:`multiline` + - :attr:`noqa` + - :attr:`previous_indent_level` + - :attr:`previous_logical` + - :attr:`previous_unindented_logical_line` + - :attr:`tokens` + - :attr:`file_tokens` + - :attr:`total_lines` + - :attr:`verbose` + """ + + def __init__(self, filename, options, lines=None): + """Initialice our file processor. + + :param str filename: + Name of the file to process + """ + self.options = options + self.filename = filename + self.lines = lines + if lines is None: + self.lines = self.read_lines() + self.strip_utf_bom() + + # Defaults for public attributes + #: Number of preceding blank lines + self.blank_before = 0 + #: Number of blank lines + self.blank_lines = 0 + #: Checker states for each plugin? + self._checker_states = {} + #: Current checker state + self.checker_state = None + #: User provided option for hang closing + self.hang_closing = options.hang_closing + #: Character used for indentation + self.indent_char = None + #: Current level of indentation + self.indent_level = 0 + #: Line number in the file + self.line_number = 0 + #: Current logical line + self.logical_line = '' + #: Maximum line length as configured by the user + self.max_line_length = options.max_line_length + #: Whether the current physical line is multiline + self.multiline = False + #: Whether or not we're observing NoQA + self.noqa = False + #: Previous level of indentation + self.previous_indent_level = 0 + #: Previous logical line + self.previous_logical = '' + #: Previous unindented (i.e. top-level) logical line + self.previous_unindented_logical_line = '' + #: Current set of tokens + self.tokens = [] + #: Total number of lines in the file + self.total_lines = len(self.lines) + #: Verbosity level of Flake8 + self.verbose = options.verbose + #: Statistics dictionary + self.statistics = { + 'logical lines': 0, + } + self._file_tokens = None + + @property + def file_tokens(self): + """The complete set of tokens for a file. + + Accessing this attribute *may* raise an InvalidSyntax exception. + + :raises: flake8.exceptions.InvalidSyntax + """ + if self._file_tokens is None: + line_iter = iter(self.lines) + try: + self._file_tokens = list(tokenize.generate_tokens( + lambda: next(line_iter) + )) + except tokenize.TokenError as exc: + raise exceptions.InvalidSyntax(exc.message, exception=exc) + + return self._file_tokens + + @contextlib.contextmanager + def inside_multiline(self, line_number): + """Context-manager to toggle the multiline attribute.""" + self.line_number = line_number + self.multiline = True + yield + self.multiline = False + + def reset_blank_before(self): + """Reset the blank_before attribute to zero.""" + self.blank_before = 0 + + def delete_first_token(self): + """Delete the first token in the list of tokens.""" + del self.tokens[0] + + def visited_new_blank_line(self): + """Note that we visited a new blank line.""" + self.blank_lines += 1 + + def update_state(self, mapping): + """Update the indent level based on the logical line mapping.""" + (start_row, start_col) = mapping[0][1] + start_line = self.lines[start_row - 1] + self.indent_level = expand_indent(start_line[:start_col]) + if self.blank_before < self.blank_lines: + self.blank_before = self.blank_lines + + def update_checker_state_for(self, plugin): + """Update the checker_state attribute for the plugin.""" + if 'checker_state' in plugin['parameters']: + self.checker_state = self._checker_states.setdefault( + plugin['name'], {} + ) + + def next_logical_line(self): + """Record the previous logical line. + + This also resets the tokens list and the blank_lines count. + """ + if self.logical_line: + self.previous_indent_level = self.indent_level + self.previous_logical = self.logical_line + if not self.indent_level: + self.previous_unindented_logical_line = self.logical_line + self.blank_lines = 0 + self.tokens = [] + self.noqa = False + + def build_logical_line_tokens(self): + """Build the mapping, comments, and logical line lists.""" + logical = [] + comments = [] + length = 0 + previous_row = previous_column = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = mutate_string(text) + if previous_row: + (start_row, start_column) = start + if previous_row != start_row: + row_index = previous_row - 1 + column_index = previous_column - 1 + previous_text = self.lines[row_index][column_index] + if (previous_text == ',' or + (previous_text not in '{[(' and + text not in '}])')): + text = ' ' + text + elif previous_column != start_column: + text = line[previous_column:start_column] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (previous_row, previous_column) = end + return comments, logical, mapping + + def build_ast(self): + """Build an abstract syntax tree from the list of lines.""" + return compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + + def build_logical_line(self): + """Build a logical line from the current tokens list.""" + comments, logical, mapping_list = self.build_logical_line_tokens() + joined_comments = ''.join(comments) + self.logical_line = ''.join(logical) + if defaults.NOQA_INLINE_REGEXP.search(joined_comments): + self.noqa = True + self.statistics['logical lines'] += 1 + return joined_comments, self.logical_line, mapping_list + + def split_line(self, token): + """Split a physical line's line based on new-lines. + + This also auto-increments the line number for the caller. + """ + for line in token[1].split('\n')[:-1]: + yield line + self.line_number += 1 + + def keyword_arguments_for(self, parameters, arguments=None): + """Generate the keyword arguments for a list of parameters.""" + if arguments is None: + arguments = {} + for param, required in parameters.items(): + if param in arguments: + continue + try: + arguments[param] = getattr(self, param) + except AttributeError as exc: + if required: + LOG.exception(exc) + raise + else: + LOG.warning('Plugin requested optional parameter "%s" ' + 'but this is not an available parameter.', + param) + return arguments + + def check_physical_error(self, error_code, line): + """Update attributes based on error code and line.""" + if error_code == 'E101': + self.indent_char = line[0] + + def generate_tokens(self): + """Tokenize the file and yield the tokens. + + :raises flake8.exceptions.InvalidSyntax: + If a :class:`tokenize.TokenError` is raised while generating + tokens. + """ + try: + for token in tokenize.generate_tokens(self.next_line): + if token[2][0] > self.total_lines: + break + self.tokens.append(token) + yield token + except (tokenize.TokenError, SyntaxError) as exc: + raise exceptions.InvalidSyntax(exception=exc) + + def line_for(self, line_number): + """Retrieve the physical line at the specified line number.""" + adjusted_line_number = line_number - 1 + # NOTE(sigmavirus24): Some plugins choose to report errors for empty + # files on Line 1. In those casese, we shouldn't bother trying to + # retrieve a physical line (since none exist). + if 0 <= adjusted_line_number < len(self.lines): + return self.lines[adjusted_line_number] + return None + + def next_line(self): + """Get the next line from the list.""" + if self.line_number >= self.total_lines: + return '' + line = self.lines[self.line_number] + self.line_number += 1 + if self.indent_char is None and line[:1] in defaults.WHITESPACE: + self.indent_char = line[0] + return line + + def read_lines(self): + # type: () -> List[str] + """Read the lines for this file checker.""" + if self.filename is None or self.filename == '-': + self.filename = self.options.stdin_display_name or 'stdin' + lines = self.read_lines_from_stdin() + else: + lines = self.read_lines_from_filename() + return lines + + def _readlines_py2(self): + # type: () -> List[str] + with open(self.filename, 'rU') as fd: + return fd.readlines() + + def _readlines_py3(self): + # type: () -> List[str] + try: + with open(self.filename, 'rb') as fd: + (coding, lines) = tokenize.detect_encoding(fd.readline) + textfd = io.TextIOWrapper(fd, coding, line_buffering=True) + return ([l.decode(coding) for l in lines] + + textfd.readlines()) + except (LookupError, SyntaxError, UnicodeError): + # If we can't detect the codec with tokenize.detect_encoding, or + # the detected encoding is incorrect, just fallback to latin-1. + with open(self.filename, encoding='latin-1') as fd: + return fd.readlines() + + def read_lines_from_filename(self): + # type: () -> List[str] + """Read the lines for a file.""" + if (2, 6) <= sys.version_info < (3, 0): + readlines = self._readlines_py2 + elif (3, 0) <= sys.version_info < (4, 0): + readlines = self._readlines_py3 + return readlines() + + def read_lines_from_stdin(self): + # type: () -> List[str] + """Read the lines from standard in.""" + return utils.stdin_get_value().splitlines(True) + + def should_ignore_file(self): + # type: () -> bool + """Check if ``# flake8: noqa`` is in the file to be ignored. + + :returns: + True if a line matches :attr:`defaults.NOQA_FILE`, + otherwise False + :rtype: + bool + """ + ignore_file = defaults.NOQA_FILE.search + return any(ignore_file(line) for line in self.lines) + + def strip_utf_bom(self): + # type: () -> NoneType + """Strip the UTF bom from the lines of the file.""" + if not self.lines: + # If we have nothing to analyze quit early + return + + first_byte = ord(self.lines[0][0]) + if first_byte not in (0xEF, 0xFEFF): + return + + # If the first byte of the file is a UTF-8 BOM, strip it + if first_byte == 0xFEFF: + self.lines[0] = self.lines[0][1:] + elif self.lines[0][:3] == '\xEF\xBB\xBF': + self.lines[0] = self.lines[0][3:] + + +def is_eol_token(token): + """Check if the token is an end-of-line token.""" + return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' + + +if COMMENT_WITH_NL: # If on Python 2.6 + def is_eol_token(token, _is_eol_token=is_eol_token): + """Check if the token is an end-of-line token.""" + return (_is_eol_token(token) or + (token[0] == tokenize.COMMENT and token[1] == token[4])) + + +def is_multiline_string(token): + """Check if this is a multiline string.""" + return token[0] == tokenize.STRING and '\n' in token[1] + + +def token_is_newline(token): + """Check if the token type is a newline token type.""" + return token[0] in NEWLINE + + +def token_is_comment(token): + """Check if the token type is a comment.""" + return COMMENT_WITH_NL and token[0] == tokenize.COMMENT + + +def count_parentheses(current_parentheses_count, token_text): + """Count the number of parentheses.""" + current_parentheses_count = current_parentheses_count or 0 + if token_text in '([{': + return current_parentheses_count + 1 + elif token_text in '}])': + return current_parentheses_count - 1 + return current_parentheses_count + + +def log_token(log, token): + """Log a token to a provided logging object.""" + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + log.log(flake8._EXTRA_VERBOSE, 'l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], + token[1])) + + +# NOTE(sigmavirus24): This was taken wholesale from +# https://github.com/PyCQA/pycodestyle +def expand_indent(line): + r"""Return the amount of indentation. + + Tabs are expanded to the next multiple of 8. + + >>> expand_indent(' ') + 4 + >>> expand_indent('\t') + 8 + >>> expand_indent(' \t') + 8 + >>> expand_indent(' \t') + 16 + """ + if '\t' not in line: + return len(line) - len(line.lstrip()) + result = 0 + for char in line: + if char == '\t': + result = result // 8 * 8 + 8 + elif char == ' ': + result += 1 + else: + break + return result + + +# NOTE(sigmavirus24): This was taken wholesale from +# https://github.com/PyCQA/pycodestyle. The in-line comments were edited to be +# more descriptive. +def mutate_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # NOTE(sigmavirus24): If there are string modifiers (e.g., b, u, r) + # use the last "character" to determine if we're using single or double + # quotes and then find the first instance of it + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Check for triple-quoted strings + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] diff --git a/contrib/flake8/reporter.py b/contrib/flake8/reporter.py deleted file mode 100644 index 1df3d9e..0000000 --- a/contrib/flake8/reporter.py +++ /dev/null @@ -1,152 +0,0 @@ -# -*- coding: utf-8 -*- -# Adapted from a contribution of Johan Dahlin - -import collections -import errno -import re -import sys -try: - import multiprocessing -except ImportError: # Python 2.5 - multiprocessing = None - -import pep8 - -__all__ = ['multiprocessing', 'BaseQReport', 'QueueReport'] - - -class BaseQReport(pep8.BaseReport): - """Base Queue Report.""" - _loaded = False # Windows support - - # Reasoning for ignored error numbers is in-line below - ignored_errors = set([ - # EPIPE: Added by sigmavirus24 - # > If output during processing is piped to something that may close - # > its own stdin before we've finished printing results, we need to - # > catch a Broken pipe error and continue on. - # > (See also: https://gitlab.com/pycqa/flake8/issues/69) - errno.EPIPE, - # NOTE(sigmavirus24): When adding to this list, include the reasoning - # on the lines before the error code and always append your error - # code. Further, please always add a trailing `,` to reduce the visual - # noise in diffs. - ]) - - def __init__(self, options): - assert options.jobs > 0 - super(BaseQReport, self).__init__(options) - self.counters = collections.defaultdict(int) - self.n_jobs = options.jobs - - # init queues - self.task_queue = multiprocessing.Queue() - self.result_queue = multiprocessing.Queue() - if sys.platform == 'win32': - # Work around http://bugs.python.org/issue10845 - sys.modules['__main__'].__file__ = __file__ - - def _cleanup_queue(self, queue): - while not queue.empty(): - queue.get_nowait() - - def _put_done(self): - # collect queues - for i in range(self.n_jobs): - self.task_queue.put('DONE') - self.update_state(self.result_queue.get()) - - def _process_main(self): - if not self._loaded: - # Windows needs to parse again the configuration - from flake8.main import get_style_guide, DEFAULT_CONFIG - get_style_guide(parse_argv=True, config_file=DEFAULT_CONFIG) - for filename in iter(self.task_queue.get, 'DONE'): - self.input_file(filename) - - def start(self): - super(BaseQReport, self).start() - self.__class__._loaded = True - # spawn processes - for i in range(self.n_jobs): - p = multiprocessing.Process(target=self.process_main) - p.daemon = True - p.start() - - def stop(self): - try: - self._put_done() - except KeyboardInterrupt: - pass - finally: - # cleanup queues to unlock threads - self._cleanup_queue(self.result_queue) - self._cleanup_queue(self.task_queue) - super(BaseQReport, self).stop() - - def process_main(self): - try: - self._process_main() - except KeyboardInterrupt: - pass - except IOError as ioerr: - # If we happen across an IOError that we aren't certain can/should - # be ignored, we should re-raise the exception. - if ioerr.errno not in self.ignored_errors: - raise - finally: - # ensure all output is flushed before main process continues - sys.stdout.flush() - sys.stderr.flush() - self.result_queue.put(self.get_state()) - - def get_state(self): - return {'total_errors': self.total_errors, - 'counters': self.counters, - 'messages': self.messages} - - def update_state(self, state): - self.total_errors += state['total_errors'] - for key, value in state['counters'].items(): - self.counters[key] += value - self.messages.update(state['messages']) - - -class FileQReport(BaseQReport): - """File Queue Report.""" - print_filename = True - - -class QueueReport(pep8.StandardReport, BaseQReport): - """Standard Queue Report.""" - - def get_file_results(self): - """Print the result and return the overall count for this file.""" - self._deferred_print.sort() - - for line_number, offset, code, text, doc in self._deferred_print: - print(self._fmt % { - 'path': self.filename, - 'row': self.line_offset + line_number, 'col': offset + 1, - 'code': code, 'text': text, - }) - # stdout is block buffered when not stdout.isatty(). - # line can be broken where buffer boundary since other processes - # write to same file. - # flush() after print() to avoid buffer boundary. - # Typical buffer size is 8192. line written safely when - # len(line) < 8192. - sys.stdout.flush() - if self._show_source: - if line_number > len(self.lines): - line = '' - else: - line = self.lines[line_number - 1] - print(line.rstrip()) - sys.stdout.flush() - print(re.sub(r'\S', ' ', line[:offset]) + '^') - sys.stdout.flush() - if self._show_pep8 and doc: - print(' ' + doc.strip()) - sys.stdout.flush() - return self.file_errors diff --git a/contrib/flake8/run.py b/contrib/flake8/run.py deleted file mode 100644 index aca929e..0000000 --- a/contrib/flake8/run.py +++ /dev/null @@ -1,11 +0,0 @@ - -""" -Implementation of the command-line I{flake8} tool. -""" -from flake8.hooks import git_hook, hg_hook # noqa -from flake8.main import check_code, check_file, Flake8Command # noqa -from flake8.main import main - - -if __name__ == '__main__': - main() diff --git a/contrib/flake8/statistics.py b/contrib/flake8/statistics.py new file mode 100644 index 0000000..d39750a --- /dev/null +++ b/contrib/flake8/statistics.py @@ -0,0 +1,129 @@ +"""Statistic collection logic for Flake8.""" +import collections + + +class Statistics(object): + """Manager of aggregated statistics for a run of Flake8.""" + + def __init__(self): + """Initialize the underlying dictionary for our statistics.""" + self._store = {} + + def error_codes(self): + """Return all unique error codes stored. + + :returns: + Sorted list of error codes. + :rtype: + list(str) + """ + return sorted({key.code for key in self._store}) + + def record(self, error): + """Add the fact that the error was seen in the file. + + :param error: + The Violation instance containing the information about the + violation. + :type error: + flake8.style_guide.Violation + """ + key = Key.create_from(error) + if key not in self._store: + self._store[key] = Statistic.create_from(error) + self._store[key].increment() + + def statistics_for(self, prefix, filename=None): + """Generate statistics for the prefix and filename. + + If you have a :class:`Statistics` object that has recorded errors, + you can generate the statistics for a prefix (e.g., ``E``, ``E1``, + ``W50``, ``W503``) with the optional filter of a filename as well. + + .. code-block:: python + + >>> stats = Statistics() + >>> stats.statistics_for('E12', + filename='src/flake8/statistics.py') + + >>> stats.statistics_for('W') + + + :param str prefix: + The error class or specific error code to find statistics for. + :param str filename: + (Optional) The filename to further filter results by. + :returns: + Generator of instances of :class:`Statistic` + """ + matching_errors = sorted(key for key in self._store + if key.matches(prefix, filename)) + for error_code in matching_errors: + yield self._store[error_code] + + +class Key(collections.namedtuple('Key', ['filename', 'code'])): + """Simple key structure for the Statistics dictionary. + + To make things clearer, easier to read, and more understandable, we use a + namedtuple here for all Keys in the underlying dictionary for the + Statistics object. + """ + + __slots__ = () + + @classmethod + def create_from(cls, error): + """Create a Key from :class:`flake8.style_guide.Violation`.""" + return cls( + filename=error.filename, + code=error.code, + ) + + def matches(self, prefix, filename): + """Determine if this key matches some constraints. + + :param str prefix: + The error code prefix that this key's error code should start with. + :param str filename: + The filename that we potentially want to match on. This can be + None to only match on error prefix. + :returns: + True if the Key's code starts with the prefix and either filename + is None, or the Key's filename matches the value passed in. + :rtype: + bool + """ + return (self.code.startswith(prefix) and + (filename is None or + self.filename == filename)) + + +class Statistic(object): + """Simple wrapper around the logic of each statistic. + + Instead of maintaining a simple but potentially hard to reason about + tuple, we create a namedtuple which has attributes and a couple + convenience methods on it. + """ + + def __init__(self, error_code, filename, message, count): + """Initialize our Statistic.""" + self.error_code = error_code + self.filename = filename + self.message = message + self.count = count + + @classmethod + def create_from(cls, error): + """Create a Statistic from a :class:`flake8.style_guide.Violation`.""" + return cls( + error_code=error.code, + filename=error.filename, + message=error.text, + count=0, + ) + + def increment(self): + """Increment the number of times we've seen this error in this file.""" + self.count += 1 diff --git a/contrib/flake8/style_guide.py b/contrib/flake8/style_guide.py new file mode 100644 index 0000000..00eb1a5 --- /dev/null +++ b/contrib/flake8/style_guide.py @@ -0,0 +1,423 @@ +"""Implementation of the StyleGuide used by Flake8.""" +import collections +import contextlib +import enum +import functools +import linecache +import logging + +from flake8 import defaults +from flake8 import statistics +from flake8 import utils + +__all__ = ( + 'StyleGuide', +) + +LOG = logging.getLogger(__name__) + + +try: + lru_cache = functools.lru_cache +except AttributeError: + def lru_cache(maxsize=128, typed=False): + """Stub for missing lru_cache.""" + def fake_decorator(func): + return func + + return fake_decorator + + +# TODO(sigmavirus24): Determine if we need to use enum/enum34 +class Selected(enum.Enum): + """Enum representing an explicitly or implicitly selected code.""" + + Explicitly = 'explicitly selected' + Implicitly = 'implicitly selected' + + +class Ignored(enum.Enum): + """Enum representing an explicitly or implicitly ignored code.""" + + Explicitly = 'explicitly ignored' + Implicitly = 'implicitly ignored' + + +class Decision(enum.Enum): + """Enum representing whether a code should be ignored or selected.""" + + Ignored = 'ignored error' + Selected = 'selected error' + + +@lru_cache(maxsize=512) +def find_noqa(physical_line): + return defaults.NOQA_INLINE_REGEXP.search(physical_line) + + +_Violation = collections.namedtuple( + 'Violation', + [ + 'code', + 'filename', + 'line_number', + 'column_number', + 'text', + 'physical_line', + ], +) + + +class Violation(_Violation): + """Class representing a violation reported by Flake8.""" + + def is_inline_ignored(self, disable_noqa): + # type: (Violation) -> bool + """Determine if an comment has been added to ignore this line. + + :param bool disable_noqa: + Whether or not users have provided ``--disable-noqa``. + :returns: + True if error is ignored in-line, False otherwise. + :rtype: + bool + """ + physical_line = self.physical_line + # TODO(sigmavirus24): Determine how to handle stdin with linecache + if disable_noqa: + return False + + if physical_line is None: + physical_line = linecache.getline(self.filename, + self.line_number) + noqa_match = find_noqa(physical_line) + if noqa_match is None: + LOG.debug('%r is not inline ignored', self) + return False + + codes_str = noqa_match.groupdict()['codes'] + if codes_str is None: + LOG.debug('%r is ignored by a blanket ``# noqa``', self) + return True + + codes = set(utils.parse_comma_separated_list(codes_str)) + if self.code in codes or self.code.startswith(tuple(codes)): + LOG.debug('%r is ignored specifically inline with ``# noqa: %s``', + self, codes_str) + return True + + LOG.debug('%r is not ignored inline with ``# noqa: %s``', + self, codes_str) + return False + + def is_in(self, diff): + """Determine if the violation is included in a diff's line ranges. + + This function relies on the parsed data added via + :meth:`~StyleGuide.add_diff_ranges`. If that has not been called and + we are not evaluating files in a diff, then this will always return + True. If there are diff ranges, then this will return True if the + line number in the error falls inside one of the ranges for the file + (and assuming the file is part of the diff data). If there are diff + ranges, this will return False if the file is not part of the diff + data or the line number of the error is not in any of the ranges of + the diff. + + :returns: + True if there is no diff or if the error is in the diff's line + number ranges. False if the error's line number falls outside + the diff's line number ranges. + :rtype: + bool + """ + if not diff: + return True + + # NOTE(sigmavirus24): The parsed diff will be a defaultdict with + # a set as the default value (if we have received it from + # flake8.utils.parse_unified_diff). In that case ranges below + # could be an empty set (which is False-y) or if someone else + # is using this API, it could be None. If we could guarantee one + # or the other, we would check for it more explicitly. + line_numbers = diff.get(self.filename) + if not line_numbers: + return False + + return self.line_number in line_numbers + + +class DecisionEngine(object): + """A class for managing the decision process around violations. + + This contains the logic for whether a violation should be reported or + ignored. + """ + + def __init__(self, options): + """Initialize the engine.""" + self.cache = {} + self.selected = tuple(options.select) + self.extended_selected = tuple(sorted( + options.extended_default_select, + reverse=True, + )) + self.enabled_extensions = tuple(options.enable_extensions) + self.all_selected = tuple(sorted( + self.selected + self.enabled_extensions, + reverse=True, + )) + self.ignored = tuple(sorted(options.ignore, reverse=True)) + self.using_default_ignore = set(self.ignored) == set(defaults.IGNORE) + self.using_default_select = ( + set(self.selected) == set(defaults.SELECT) + ) + + def _in_all_selected(self, code): + return self.all_selected and code.startswith(self.all_selected) + + def _in_extended_selected(self, code): + return (self.extended_selected and + code.startswith(self.extended_selected)) + + def was_selected(self, code): + # type: (str) -> Union[Selected, Ignored] + """Determine if the code has been selected by the user. + + :param str code: + The code for the check that has been run. + :returns: + Selected.Implicitly if the selected list is empty, + Selected.Explicitly if the selected list is not empty and a match + was found, + Ignored.Implicitly if the selected list is not empty but no match + was found. + """ + if self._in_all_selected(code): + return Selected.Explicitly + + if not self.all_selected and self._in_extended_selected(code): + # If it was not explicitly selected, it may have been implicitly + # selected because the check comes from a plugin that is enabled by + # default + return Selected.Implicitly + + return Ignored.Implicitly + + def was_ignored(self, code): + # type: (str) -> Union[Selected, Ignored] + """Determine if the code has been ignored by the user. + + :param str code: + The code for the check that has been run. + :returns: + Selected.Implicitly if the ignored list is empty, + Ignored.Explicitly if the ignored list is not empty and a match was + found, + Selected.Implicitly if the ignored list is not empty but no match + was found. + """ + if self.ignored and code.startswith(self.ignored): + return Ignored.Explicitly + + return Selected.Implicitly + + def more_specific_decision_for(self, code): + # type: (Violation) -> Decision + select = find_first_match(code, self.all_selected) + extra_select = find_first_match(code, self.extended_selected) + ignore = find_first_match(code, self.ignored) + + if select and ignore: + # If the violation code appears in both the select and ignore + # lists (in some fashion) then if we're using the default ignore + # list and a custom select list we should select the code. An + # example usage looks like this: + # A user has a code that would generate an E126 violation which + # is in our default ignore list and they specify select=E. + # We should be reporting that violation. This logic changes, + # however, if they specify select and ignore such that both match. + # In that case we fall through to our find_more_specific call. + # If, however, the user hasn't specified a custom select, and + # we're using the defaults for both select and ignore then the + # more specific rule must win. In most cases, that will be to + # ignore the violation since our default select list is very + # high-level and our ignore list is highly specific. + if self.using_default_ignore and not self.using_default_select: + return Decision.Selected + return find_more_specific(select, ignore) + if extra_select and ignore: + # At this point, select is false-y. Now we need to check if the + # code is in our extended select list and our ignore list. This is + # a *rare* case as we see little usage of the extended select list + # that plugins can use, so I suspect this section may change to + # look a little like the block above in which we check if we're + # using our default ignore list. + return find_more_specific(extra_select, ignore) + if select or (extra_select and self.using_default_select): + # Here, ignore was false-y and the user has either selected + # explicitly the violation or the violation is covered by + # something in the extended select list and we're using the + # default select list. In either case, we want the violation to be + # selected. + return Decision.Selected + if (select is None and + (extra_select is None or not self.using_default_ignore)): + return Decision.Ignored + if ((select is None and not self.using_default_select) and + (ignore is None and self.using_default_ignore)): + return Decision.Ignored + return Decision.Selected + + def make_decision(self, code): + """Decide if code should be ignored or selected.""" + LOG.debug('Deciding if "%s" should be reported', code) + selected = self.was_selected(code) + ignored = self.was_ignored(code) + LOG.debug('The user configured "%s" to be "%s", "%s"', + code, selected, ignored) + + if ((selected is Selected.Explicitly or + selected is Selected.Implicitly) and + ignored is Selected.Implicitly): + decision = Decision.Selected + elif ((selected is Selected.Explicitly and + ignored is Ignored.Explicitly) or + (selected is Ignored.Implicitly and + ignored is Selected.Implicitly)): + decision = self.more_specific_decision_for(code) + elif (selected is Ignored.Implicitly or + ignored is Ignored.Explicitly): + decision = Decision.Ignored # pylint: disable=R0204 + return decision + + def decision_for(self, code): + # type: (str) -> Decision + """Return the decision for a specific code. + + This method caches the decisions for codes to avoid retracing the same + logic over and over again. We only care about the select and ignore + rules as specified by the user in their configuration files and + command-line flags. + + This method does not look at whether the specific line is being + ignored in the file itself. + + :param str code: + The code for the check that has been run. + """ + decision = self.cache.get(code) + if decision is None: + decision = self.make_decision(code) + self.cache[code] = decision + LOG.debug('"%s" will be "%s"', code, decision) + return decision + + +class StyleGuide(object): + """Manage a Flake8 user's style guide.""" + + def __init__(self, options, listener_trie, formatter, decider=None): + """Initialize our StyleGuide. + + .. todo:: Add parameter documentation. + """ + self.options = options + self.listener = listener_trie + self.formatter = formatter + self.stats = statistics.Statistics() + self.decider = decider or DecisionEngine(options) + self._parsed_diff = {} + + @contextlib.contextmanager + def processing_file(self, filename): + """Record the fact that we're processing the file's results.""" + self.formatter.beginning(filename) + yield self + self.formatter.finished(filename) + + def should_report_error(self, code): + # type: (str) -> Decision + """Determine if the error code should be reported or ignored. + + This method only cares about the select and ignore rules as specified + by the user in their configuration files and command-line flags. + + This method does not look at whether the specific line is being + ignored in the file itself. + + :param str code: + The code for the check that has been run. + """ + return self.decider.decision_for(code) + + def handle_error(self, code, filename, line_number, column_number, text, + physical_line=None): + # type: (str, str, int, int, str) -> int + """Handle an error reported by a check. + + :param str code: + The error code found, e.g., E123. + :param str filename: + The file in which the error was found. + :param int line_number: + The line number (where counting starts at 1) at which the error + occurs. + :param int column_number: + The column number (where counting starts at 1) at which the error + occurs. + :param str text: + The text of the error message. + :param str physical_line: + The actual physical line causing the error. + :returns: + 1 if the error was reported. 0 if it was ignored. This is to allow + for counting of the number of errors found that were not ignored. + :rtype: + int + """ + disable_noqa = self.options.disable_noqa + # NOTE(sigmavirus24): Apparently we're provided with 0-indexed column + # numbers so we have to offset that here. Also, if a SyntaxError is + # caught, column_number may be None. + if not column_number: + column_number = 0 + error = Violation(code, filename, line_number, column_number + 1, + text, physical_line) + error_is_selected = (self.should_report_error(error.code) is + Decision.Selected) + is_not_inline_ignored = error.is_inline_ignored(disable_noqa) is False + is_included_in_diff = error.is_in(self._parsed_diff) + if (error_is_selected and is_not_inline_ignored and + is_included_in_diff): + self.formatter.handle(error) + self.stats.record(error) + self.listener.notify(error.code, error) + return 1 + return 0 + + def add_diff_ranges(self, diffinfo): + """Update the StyleGuide to filter out information not in the diff. + + This provides information to the StyleGuide so that only the errors + in the line number ranges are reported. + + :param dict diffinfo: + Dictionary mapping filenames to sets of line number ranges. + """ + self._parsed_diff = diffinfo + + +def find_more_specific(selected, ignored): + if selected.startswith(ignored) and selected != ignored: + return Decision.Selected + return Decision.Ignored + + +def find_first_match(error_code, code_list): + startswith = error_code.startswith + for code in code_list: + if startswith(code): + break + else: + return None + return code diff --git a/contrib/flake8/tests/__init__.py b/contrib/flake8/tests/__init__.py deleted file mode 100644 index 792d600..0000000 --- a/contrib/flake8/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/contrib/flake8/tests/_test_warnings.py b/contrib/flake8/tests/_test_warnings.py deleted file mode 100644 index 004a597..0000000 --- a/contrib/flake8/tests/_test_warnings.py +++ /dev/null @@ -1,309 +0,0 @@ -""" - _test_warnings.py - - Tests for the warnings that are emitted by flake8. - - This module is named _test_warnings instead of test_warnings so that a - normal nosetests run does not collect it. The tests in this module pass - when they are run alone, but they fail when they are run along with other - tests (nosetests --with-isolation doesn't help). - - In tox.ini, these tests are run separately. - -""" - -from __future__ import with_statement - -import os -import warnings -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import engine -from flake8.util import is_windows - -# The Problem -# ------------ -# -# Some of the tests in this module pass when this module is run on its own, but -# they fail when this module is run as part of the whole test suite. These are -# the problematic tests: -# -# test_jobs_verbose -# test_stdin_jobs_warning -# -# On some platforms, the warnings.capture_warnings function doesn't work -# properly when run with the other flake8 tests. It drops some warnings, even -# though the warnings filter is set to 'always'. However, when run separately, -# these tests pass. -# -# This problem only occurs on Windows, with Python 3.3 and older. Maybe it's -# related to PEP 446 - Inheritable file descriptors? -# -# -# -# -# Things that didn't work -# ------------ -# -# Nose --attr -# I tried using the nosetests --attr feature to run the tests separately. I -# put the following in setup.cfg -# -# [nosetests] -# atttr=!run_alone -# -# Then I added a tox section thst did this -# -# nosetests --attr=run_alone -# -# However, the command line --attr would not override the config file --attr, -# so the special tox section wound up runing all the tests, and failing. -# -# -# -# Nose --with-isolation -# The nosetests --with-isolation flag did not help. -# -# -# -# unittest.skipIf -# I tried decorating the problematic tests with the unittest.skipIf -# decorator. -# -# @unittest.skipIf(is_windows() and sys.version_info < (3, 4), -# "Fails on Windows with Python < 3.4 when run with other" -# " tests.") -# -# The idea is, skip the tests in the main test run, on affected platforms. -# Then, only on those platforms, come back in later and run the tests -# separately. -# -# I added a new stanza to tox.ini, to run the tests separately on the -# affected platforms. -# -# nosetests --no-skip -# -# I ran in to a bug in the nosetests skip plugin. It would report the test as -# having been run, but it would not actually run the test. So, when run with -# --no-skip, the following test would be reported as having run and passed! -# -# @unittest.skip("This passes o_o") -# def test_should_fail(self): -# assert 0 -# -# This bug has been reported here: -# "--no-skip broken with Python 2.7" -# https://github.com/nose-devs/nose/issues/512 -# -# -# -# py.test -# -# I tried using py.test, and its @pytest.mark.xfail decorator. I added some -# separate stanzas in tox, and useing the pytest --runxfail option to run the -# tests separately. This allows us to run all the tests together, on -# platforms that allow it. On platforms that don't allow us to run the tests -# all together, this still runs all the tests, but in two separate steps. -# -# This is the same solution as the nosetests --no-skip solution I described -# above, but --runxfail does not have the same bug as --no-skip. -# -# This has the advantage that all tests are discoverable by default, outside -# of tox. However, nose does not recognize the pytest.mark.xfail decorator. -# So, if a user runs nosetests, it still tries to run the problematic tests -# together with the rest of the test suite, causing them to fail. -# -# -# -# -# -# -# Solution -# ------------ -# Move the problematic tests to _test_warnings.py, so nose.collector will not -# find them. Set up a separate section in tox.ini that runs this: -# -# nosetests flake8.tests._test_warnings -# -# This allows all tests to pass on all platforms, when run through tox. -# However, it means that, even on unaffected platforms, the problematic tests -# are not discovered and run outside of tox (if the user just runs nosetests -# manually, for example). - - -class IntegrationTestCaseWarnings(unittest.TestCase): - """Integration style tests to check that warnings are issued properly for - different command line options.""" - - windows_warning_text = ("The --jobs option is not available on Windows." - " Ignoring --jobs arguments.") - stdin_warning_text = ("The --jobs option is not compatible with" - " supplying input using - . Ignoring --jobs" - " arguments.") - - def this_file(self): - """Return the real path of this file.""" - this_file = os.path.realpath(__file__) - if this_file.endswith("pyc"): - this_file = this_file[:-1] - return this_file - - @staticmethod - def get_style_guide_with_warnings(engine, *args, **kwargs): - """ - Return a style guide object (obtained by calling - engine.get_style_guide) and a list of the warnings that were raised in - the process. - - Note: not threadsafe - """ - - # Note - # https://docs.python.org/2/library/warnings.html - # - # The catch_warnings manager works by replacing and then later - # restoring the module's showwarning() function and internal list of - # filter specifications. This means the context manager is modifying - # global state and therefore is not thread-safe - - with warnings.catch_warnings(record=True) as collected_warnings: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - - # Get the style guide - style_guide = engine.get_style_guide(*args, **kwargs) - - # Now that the warnings have been collected, return the style guide and - # the warnings. - return (style_guide, collected_warnings) - - def verify_warnings(self, collected_warnings, expected_warnings): - """ - Verifies that collected_warnings is a sequence that contains user - warnings that match the sequence of string values passed in as - expected_warnings. - """ - if expected_warnings is None: - expected_warnings = [] - - collected_user_warnings = [w for w in collected_warnings - if issubclass(w.category, UserWarning)] - - self.assertEqual(len(collected_user_warnings), - len(expected_warnings)) - - collected_warnings_set = set(str(warning.message) - for warning - in collected_user_warnings) - expected_warnings_set = set(expected_warnings) - self.assertEqual(collected_warnings_set, expected_warnings_set) - - def check_files_collect_warnings(self, - arglist=[], - explicit_stdin=False, - count=0, - verbose=False): - """Call check_files and collect any warnings that are issued.""" - if verbose: - arglist.append('--verbose') - if explicit_stdin: - target_file = "-" - else: - target_file = self.this_file() - argv = ['flake8'] + arglist + [target_file] - with mock.patch("sys.argv", argv): - (style_guide, - collected_warnings, - ) = self.get_style_guide_with_warnings(engine, - parse_argv=True) - report = style_guide.check_files() - self.assertEqual(report.total_errors, count) - return style_guide, report, collected_warnings - - def check_files_no_warnings_allowed(self, - arglist=[], - explicit_stdin=False, - count=0, - verbose=False): - """Call check_files, and assert that there were no warnings issued.""" - (style_guide, - report, - collected_warnings, - ) = self.check_files_collect_warnings(arglist=arglist, - explicit_stdin=explicit_stdin, - count=count, - verbose=verbose) - self.verify_warnings(collected_warnings, expected_warnings=None) - return style_guide, report - - def _job_tester(self, jobs, verbose=False): - # mock stdout.flush so we can count the number of jobs created - with mock.patch('sys.stdout.flush') as mocked: - (guide, - report, - collected_warnings, - ) = self.check_files_collect_warnings( - arglist=['--jobs=%s' % jobs], - verbose=verbose) - - if is_windows(): - # The code path where guide.options.jobs gets converted to an - # int is not run on windows. So, do the int conversion here. - self.assertEqual(int(guide.options.jobs), jobs) - # On windows, call count is always zero. - self.assertEqual(mocked.call_count, 0) - else: - self.assertEqual(guide.options.jobs, jobs) - self.assertEqual(mocked.call_count, jobs) - - expected_warings = [] - if verbose and is_windows(): - expected_warings.append(self.windows_warning_text) - self.verify_warnings(collected_warnings, expected_warings) - - def test_jobs(self, verbose=False): - self._job_tester(2, verbose=verbose) - self._job_tester(10, verbose=verbose) - - def test_no_args_no_warnings(self, verbose=False): - self.check_files_no_warnings_allowed(verbose=verbose) - - def test_stdin_jobs_warning(self, verbose=False): - self.count = 0 - - def fake_stdin(): - self.count += 1 - with open(self.this_file(), "r") as f: - return f.read() - - with mock.patch("pep8.stdin_get_value", fake_stdin): - (style_guide, - report, - collected_warnings, - ) = self.check_files_collect_warnings(arglist=['--jobs=4'], - explicit_stdin=True, - verbose=verbose) - expected_warings = [] - if verbose: - expected_warings.append(self.stdin_warning_text) - if is_windows(): - expected_warings.append(self.windows_warning_text) - self.verify_warnings(collected_warnings, expected_warings) - self.assertEqual(self.count, 1) - - def test_jobs_verbose(self): - self.test_jobs(verbose=True) - - def test_no_args_no_warnings_verbose(self): - self.test_no_args_no_warnings(verbose=True) - - def test_stdin_jobs_warning_verbose(self): - self.test_stdin_jobs_warning(verbose=True) - - -if __name__ == '__main__': - unittest.main() diff --git a/contrib/flake8/tests/test_engine.py b/contrib/flake8/tests/test_engine.py deleted file mode 100644 index 2afabbb..0000000 --- a/contrib/flake8/tests/test_engine.py +++ /dev/null @@ -1,236 +0,0 @@ -from __future__ import with_statement - -import errno -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import engine, util, __version__, reporter -import pep8 - - -class TestEngine(unittest.TestCase): - def setUp(self): - self.patches = {} - - def tearDown(self): - assert len(self.patches.items()) == 0 - - def start_patch(self, patch): - self.patches[patch] = mock.patch(patch) - return self.patches[patch].start() - - def stop_patches(self): - patches = self.patches.copy() - for k, v in patches.items(): - v.stop() - del(self.patches[k]) - - def test_get_style_guide(self): - with mock.patch('flake8.engine._register_extensions') as reg_ext: - reg_ext.return_value = ([], [], [], []) - g = engine.get_style_guide() - self.assertTrue(isinstance(g, engine.StyleGuide)) - reg_ext.assert_called_once_with() - - def test_get_style_guide_kwargs(self): - m = mock.Mock() - with mock.patch('flake8.engine.StyleGuide') as StyleGuide: - with mock.patch('flake8.engine.get_parser') as get_parser: - m.ignored_extensions = [] - StyleGuide.return_value.options.jobs = '42' - StyleGuide.return_value.options.diff = False - get_parser.return_value = (m, []) - engine.get_style_guide(foo='bar') - get_parser.assert_called_once_with() - StyleGuide.assert_called_once_with(**{'parser': m, 'foo': 'bar'}) - - def test_register_extensions(self): - with mock.patch('pep8.register_check') as register_check: - registered_exts = engine._register_extensions() - self.assertTrue(isinstance(registered_exts[0], util.OrderedSet)) - self.assertTrue(len(registered_exts[0]) > 0) - for i in registered_exts[1:]: - self.assertTrue(isinstance(i, list)) - self.assertTrue(register_check.called) - - def test_disable_extensions(self): - parser = mock.MagicMock() - options = mock.MagicMock() - - parser.ignored_extensions = ['I123', 'I345', 'I678', 'I910'] - - options.enabled_extensions = 'I345,\nI678,I910' - options.ignore = ('E121', 'E123') - - engine._disable_extensions(parser, options) - self.assertEqual(set(options.ignore), set(['E121', 'E123', 'I123'])) - - def test_get_parser(self): - # setup - re = self.start_patch('flake8.engine._register_extensions') - gpv = self.start_patch('flake8.engine.get_python_version') - pgp = self.start_patch('pep8.get_parser') - m = mock.Mock() - re.return_value = ([('pyflakes', '0.7'), ('mccabe', '0.2')], [], [], - []) - gpv.return_value = 'Python Version' - pgp.return_value = m - # actual call we're testing - parser, hooks = engine.get_parser() - # assertions - self.assertTrue(re.called) - self.assertTrue(gpv.called) - pgp.assert_called_once_with( - 'flake8', - '%s (pyflakes: 0.7, mccabe: 0.2) Python Version' % __version__) - self.assertTrue(m.remove_option.called) - self.assertTrue(m.add_option.called) - self.assertEqual(parser, m) - self.assertEqual(hooks, []) - # clean-up - self.stop_patches() - - def test_get_python_version(self): - self.assertTrue('on' in engine.get_python_version()) - # Silly test but it will provide 100% test coverage - # Also we can never be sure (without reconstructing the string - # ourselves) what system we may be testing on. - - def test_windows_disables_jobs(self): - with mock.patch('flake8.util.is_windows') as is_windows: - is_windows.return_value = True - guide = engine.get_style_guide() - assert isinstance(guide, reporter.BaseQReport) is False - - def test_stdin_disables_jobs(self): - with mock.patch('flake8.util.is_using_stdin') as is_using_stdin: - is_using_stdin.return_value = True - guide = engine.get_style_guide() - assert isinstance(guide, reporter.BaseQReport) is False - - def test_disables_extensions_that_are_not_selected(self): - with mock.patch('flake8.engine._register_extensions') as re: - re.return_value = ([('fake_ext', '0.1a1')], [], [], ['X']) - sg = engine.get_style_guide() - assert 'X' in sg.options.ignore - - def test_enables_off_by_default_extensions(self): - with mock.patch('flake8.engine._register_extensions') as re: - re.return_value = ([('fake_ext', '0.1a1')], [], [], ['X']) - parser, options = engine.get_parser() - parser.parse_args(['--select=X']) - sg = engine.StyleGuide(parser=parser) - assert 'X' not in sg.options.ignore - - def test_load_entry_point_verifies_requirements(self): - entry_point = mock.Mock(spec=['require', 'resolve', 'load']) - - engine._load_entry_point(entry_point, verify_requirements=True) - entry_point.require.assert_called_once_with() - entry_point.resolve.assert_called_once_with() - - def test_load_entry_point_does_not_verify_requirements(self): - entry_point = mock.Mock(spec=['require', 'resolve', 'load']) - - engine._load_entry_point(entry_point, verify_requirements=False) - self.assertFalse(entry_point.require.called) - entry_point.resolve.assert_called_once_with() - - def test_load_entry_point_passes_require_argument_to_load(self): - entry_point = mock.Mock(spec=['load']) - - engine._load_entry_point(entry_point, verify_requirements=True) - entry_point.load.assert_called_once_with(require=True) - entry_point.reset_mock() - - engine._load_entry_point(entry_point, verify_requirements=False) - entry_point.load.assert_called_once_with(require=False) - - -def oserror_generator(error_number, message='Ominous OSError message'): - def oserror_side_effect(*args, **kwargs): - if hasattr(oserror_side_effect, 'used'): - return - - oserror_side_effect.used = True - raise OSError(error_number, message) - - return oserror_side_effect - - -class TestStyleGuide(unittest.TestCase): - def setUp(self): - mocked_styleguide = mock.Mock(spec=engine.NoQAStyleGuide) - self.styleguide = engine.StyleGuide(styleguide=mocked_styleguide) - self.mocked_sg = mocked_styleguide - - def test_proxies_excluded(self): - self.styleguide.excluded('file.py', parent='.') - - self.mocked_sg.excluded.assert_called_once_with('file.py', parent='.') - - def test_proxies_init_report(self): - reporter = object() - self.styleguide.init_report(reporter) - - self.mocked_sg.init_report.assert_called_once_with(reporter) - - def test_proxies_check_files(self): - self.styleguide.check_files(['foo', 'bar']) - - self.mocked_sg.check_files.assert_called_once_with( - paths=['foo', 'bar'] - ) - - def test_proxies_input_file(self): - self.styleguide.input_file('file.py', - lines=[9, 10], - expected='foo', - line_offset=20) - - self.mocked_sg.input_file.assert_called_once_with(filename='file.py', - lines=[9, 10], - expected='foo', - line_offset=20) - - def test_check_files_retries_on_specific_OSErrors(self): - self.mocked_sg.check_files.side_effect = oserror_generator( - errno.ENOSPC, 'No space left on device' - ) - - self.styleguide.check_files(['foo', 'bar']) - - self.mocked_sg.init_report.assert_called_once_with(pep8.StandardReport) - - def test_input_file_retries_on_specific_OSErrors(self): - self.mocked_sg.input_file.side_effect = oserror_generator( - errno.ENOSPC, 'No space left on device' - ) - - self.styleguide.input_file('file.py') - - self.mocked_sg.init_report.assert_called_once_with(pep8.StandardReport) - - def test_check_files_reraises_unknown_OSErrors(self): - self.mocked_sg.check_files.side_effect = oserror_generator( - errno.EADDRINUSE, - 'lol why are we talking about binding to sockets' - ) - - self.assertRaises(OSError, self.styleguide.check_files, - ['foo', 'bar']) - - def test_input_file_reraises_unknown_OSErrors(self): - self.mocked_sg.input_file.side_effect = oserror_generator( - errno.EADDRINUSE, - 'lol why are we talking about binding to sockets' - ) - - self.assertRaises(OSError, self.styleguide.input_file, - ['foo', 'bar']) - -if __name__ == '__main__': - unittest.main() diff --git a/contrib/flake8/tests/test_hooks.py b/contrib/flake8/tests/test_hooks.py deleted file mode 100644 index ba46794..0000000 --- a/contrib/flake8/tests/test_hooks.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Module containing the tests for flake8.hooks.""" -import os -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -import flake8.hooks -from flake8.util import is_windows - - -def excluded(filename): - return filename.endswith('afile.py') - - -class TestGitHook(unittest.TestCase): - if is_windows: - # On Windows, absolute paths start with a drive letter, for example C: - # Here we build a fake absolute path starting with the current drive - # letter, for example C:\fake\temp - current_drive, ignore_tail = os.path.splitdrive(os.getcwd()) - fake_abs_path = os.path.join(current_drive, os.path.sep, 'fake', 'tmp') - else: - fake_abs_path = os.path.join(os.path.sep, 'fake', 'tmp') - - @mock.patch('os.makedirs') - @mock.patch('flake8.hooks.open', create=True) - @mock.patch('shutil.rmtree') - @mock.patch('tempfile.mkdtemp', return_value=fake_abs_path) - @mock.patch('flake8.hooks.run', - return_value=(None, - [os.path.join('foo', 'afile.py'), - os.path.join('foo', 'bfile.py')], - None)) - @mock.patch('flake8.hooks.get_style_guide') - def test_prepends_tmp_directory_to_exclude(self, get_style_guide, run, - *args): - style_guide = get_style_guide.return_value = mock.Mock() - style_guide.options.exclude = [os.path.join('foo', 'afile.py')] - style_guide.options.filename = [os.path.join('foo', '*')] - style_guide.excluded = excluded - - flake8.hooks.git_hook() - - dirname, filename = os.path.split( - os.path.abspath(os.path.join('foo', 'bfile.py'))) - if is_windows: - # In Windows, the absolute path in dirname will start with a drive - # letter. Here, we discad the drive letter. - ignore_drive, dirname = os.path.splitdrive(dirname) - tmpdir = os.path.join(self.fake_abs_path, dirname[1:]) - tmpfile = os.path.join(tmpdir, 'bfile.py') - style_guide.check_files.assert_called_once_with([tmpfile]) - - -if __name__ == '__main__': - unittest.main() diff --git a/contrib/flake8/tests/test_integration.py b/contrib/flake8/tests/test_integration.py deleted file mode 100644 index d1417c6..0000000 --- a/contrib/flake8/tests/test_integration.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import with_statement - -import os -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import engine -from flake8.util import is_windows - - -class IntegrationTestCase(unittest.TestCase): - """Integration style tests to exercise different command line options.""" - - def this_file(self): - """Return the real path of this file.""" - this_file = os.path.realpath(__file__) - if this_file.endswith("pyc"): - this_file = this_file[:-1] - return this_file - - def check_files(self, arglist=[], explicit_stdin=False, count=0): - """Call check_files.""" - if explicit_stdin: - target_file = "-" - else: - target_file = self.this_file() - argv = ['flake8'] + arglist + [target_file] - with mock.patch("sys.argv", argv): - style_guide = engine.get_style_guide(parse_argv=True) - report = style_guide.check_files() - self.assertEqual(report.total_errors, count) - return style_guide, report - - def test_no_args(self): - # assert there are no reported errors - self.check_files() - - def _job_tester(self, jobs): - # mock stdout.flush so we can count the number of jobs created - with mock.patch('sys.stdout.flush') as mocked: - guide, report = self.check_files(arglist=['--jobs=%s' % jobs]) - if is_windows(): - # The code path where guide.options.jobs gets converted to an - # int is not run on windows. So, do the int conversion here. - self.assertEqual(int(guide.options.jobs), jobs) - # On windows, call count is always zero. - self.assertEqual(mocked.call_count, 0) - else: - self.assertEqual(guide.options.jobs, jobs) - self.assertEqual(mocked.call_count, jobs) - - def test_jobs(self): - self._job_tester(2) - self._job_tester(10) - - def test_stdin(self): - self.count = 0 - - def fake_stdin(): - self.count += 1 - with open(self.this_file(), "r") as f: - return f.read() - - with mock.patch("pep8.stdin_get_value", fake_stdin): - guide, report = self.check_files(arglist=['--jobs=4'], - explicit_stdin=True) - self.assertEqual(self.count, 1) - - def test_stdin_fail(self): - def fake_stdin(): - return "notathing\n" - with mock.patch("pep8.stdin_get_value", fake_stdin): - # only assert needed is in check_files - guide, report = self.check_files(arglist=['--jobs=4'], - explicit_stdin=True, - count=1) diff --git a/contrib/flake8/tests/test_main.py b/contrib/flake8/tests/test_main.py deleted file mode 100644 index af08093..0000000 --- a/contrib/flake8/tests/test_main.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import with_statement - -import unittest - -import setuptools -from flake8 import main - - -class TestMain(unittest.TestCase): - def test_issue_39_regression(self): - distribution = setuptools.Distribution() - cmd = main.Flake8Command(distribution) - cmd.options_dict = {} - cmd.run() - - -if __name__ == '__main__': - unittest.main() diff --git a/contrib/flake8/tests/test_pyflakes.py b/contrib/flake8/tests/test_pyflakes.py deleted file mode 100644 index fb2f042..0000000 --- a/contrib/flake8/tests/test_pyflakes.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import with_statement - -import ast -import unittest - -from collections import namedtuple - -from flake8._pyflakes import FlakesChecker - -Options = namedtuple("Options", ['builtins', 'doctests', - 'include_in_doctest', - 'exclude_from_doctest']) - - -class TestFlakesChecker(unittest.TestCase): - - def setUp(self): - self.tree = ast.parse('print("cookies")') - - def test_doctest_flag_enabled(self): - options = Options(builtins=None, doctests=True, - include_in_doctest='', - exclude_from_doctest='') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, 'cookies.txt') - assert flake_checker.withDoctest is True - - def test_doctest_flag_disabled(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='', - exclude_from_doctest='') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, 'cookies.txt') - assert flake_checker.withDoctest is False - - def test_doctest_flag_enabled_exclude_file(self): - options = Options(builtins=None, doctests=True, - include_in_doctest='', - exclude_from_doctest='cookies.txt,' - 'hungry/cookies.txt') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is False - - def test_doctest_flag_disabled_include_file(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./cookies.txt,cake_yuck.txt', - exclude_from_doctest='') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is True - - def test_doctest_flag_disabled_include_file_exclude_dir(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./cookies.txt', - exclude_from_doctest='./') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is True - - def test_doctest_flag_disabled_include_dir_exclude_file(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./', - exclude_from_doctest='./cookies.txt') - FlakesChecker.parse_options(options) - flake_checker = FlakesChecker(self.tree, './cookies.txt') - assert flake_checker.withDoctest is False - - def test_doctest_flag_disabled_include_file_exclude_file_error(self): - options = Options(builtins=None, doctests=False, - include_in_doctest='./cookies.txt', - exclude_from_doctest='./cookies.txt,cake_yuck.txt') - self.assertRaises(ValueError, FlakesChecker.parse_options, options) diff --git a/contrib/flake8/tests/test_reporter.py b/contrib/flake8/tests/test_reporter.py deleted file mode 100644 index f91bb52..0000000 --- a/contrib/flake8/tests/test_reporter.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import with_statement - -import errno -import unittest -try: - from unittest import mock -except ImportError: - import mock # < PY33 - -from flake8 import reporter - - -def ioerror_report_factory(errno_code): - class IOErrorBaseQReport(reporter.BaseQReport): - def _process_main(self): - raise IOError(errno_code, 'Fake bad pipe exception') - - options = mock.MagicMock() - options.jobs = 2 - return IOErrorBaseQReport(options) - - -class TestBaseQReport(unittest.TestCase): - def test_does_not_raise_a_bad_pipe_ioerror(self): - """Test that no EPIPE IOError exception is re-raised or leaked.""" - report = ioerror_report_factory(errno.EPIPE) - try: - report.process_main() - except IOError: - self.fail('BaseQReport.process_main raised an IOError for EPIPE' - ' but it should have caught this exception.') - - def test_raises_a_enoent_ioerror(self): - """Test that an ENOENT IOError exception is re-raised.""" - report = ioerror_report_factory(errno.ENOENT) - self.assertRaises(IOError, report.process_main) diff --git a/contrib/flake8/tests/test_util.py b/contrib/flake8/tests/test_util.py deleted file mode 100644 index 32a1d44..0000000 --- a/contrib/flake8/tests/test_util.py +++ /dev/null @@ -1,86 +0,0 @@ -import unittest - -from flake8.util import option_normalizer - - -class TestOptionSerializer(unittest.TestCase): - - def test_1_is_true(self): - option = option_normalizer('1') - self.assertTrue(option) - - def test_T_is_true(self): - option = option_normalizer('T') - self.assertTrue(option) - - def test_TRUE_is_true(self): - option = option_normalizer('TRUE') - self.assertTrue(option, True) - - def test_ON_is_true(self): - option = option_normalizer('ON') - self.assertTrue(option) - - def test_t_is_true(self): - option = option_normalizer('t') - self.assertTrue(option) - - def test_true_is_true(self): - option = option_normalizer('true') - self.assertTrue(option) - - def test_on_is_true(self): - option = option_normalizer('on') - self.assertTrue(option) - - def test_0_is_false(self): - option = option_normalizer('0') - self.assertFalse(option) - - def test_F_is_false(self): - option = option_normalizer('F') - self.assertFalse(option) - - def test_FALSE_is_false(self): - option = option_normalizer('FALSE') - self.assertFalse(option) - - def test_OFF_is_false(self): - option = option_normalizer('OFF') - self.assertFalse(option) - - def test_f_is_false(self): - option = option_normalizer('f') - self.assertFalse(option) - - def test_false_is_false(self): - option = option_normalizer('false') - self.assertFalse(option) - - def test_off_is_false(self): - option = option_normalizer('off') - self.assertFalse(option) - - def test_parses_lists(self): - answer = ['F401', 'F402', 'F403', 'F404'] - - option = option_normalizer('F401,F402,F403,F404') - self.assertEqual(option, answer) - - option = option_normalizer('F401 ,F402 ,F403 ,F404') - self.assertEqual(option, answer) - - option = option_normalizer('F401, F402, F403, F404') - self.assertEqual(option, answer) - - option = option_normalizer('''\ - F401, - F402, - F403, - F404, - ''') - self.assertEqual(option, answer) - - -if __name__ == '__main__': - unittest.main() diff --git a/contrib/flake8/util.py b/contrib/flake8/util.py deleted file mode 100644 index da33f42..0000000 --- a/contrib/flake8/util.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -import os - -try: - import ast - iter_child_nodes = ast.iter_child_nodes -except ImportError: # Python 2.5 - import _ast as ast - - if 'decorator_list' not in ast.ClassDef._fields: - # Patch the missing attribute 'decorator_list' - ast.ClassDef.decorator_list = () - ast.FunctionDef.decorator_list = property(lambda s: s.decorators) - - def iter_child_nodes(node): - """ - Yield all direct child nodes of *node*, that is, all fields that - are nodes and all items of fields that are lists of nodes. - """ - if not node._fields: - return - for name in node._fields: - field = getattr(node, name, None) - if isinstance(field, ast.AST): - yield field - elif isinstance(field, list): - for item in field: - if isinstance(item, ast.AST): - yield item - - -class OrderedSet(list): - """List without duplicates.""" - __slots__ = () - - def add(self, value): - if value not in self: - self.append(value) - - -def is_windows(): - """Determine if the system is Windows.""" - return os.name == 'nt' - - -def is_using_stdin(paths): - """Determine if we're running checks on stdin.""" - return '-' in paths - - -def warn_when_using_jobs(options): - return (options.verbose and options.jobs and options.jobs.isdigit() and - int(options.jobs) > 1) - - -def force_disable_jobs(styleguide): - return is_windows() or is_using_stdin(styleguide.paths) - - -def option_normalizer(value): - if str(value).upper() in ('1', 'T', 'TRUE', 'ON'): - value = True - if str(value).upper() in ('0', 'F', 'FALSE', 'OFF'): - value = False - - if isinstance(value, str): - value = [opt.strip() for opt in value.split(',') if opt.strip()] - - return value diff --git a/contrib/flake8/utils.py b/contrib/flake8/utils.py new file mode 100644 index 0000000..d28b810 --- /dev/null +++ b/contrib/flake8/utils.py @@ -0,0 +1,345 @@ +"""Utility methods for flake8.""" +import collections +import fnmatch as _fnmatch +import inspect +import io +import os +import platform +import re +import sys +import tokenize + +DIFF_HUNK_REGEXP = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') +COMMA_SEPARATED_LIST_RE = re.compile(r'[,\s]') +LOCAL_PLUGIN_LIST_RE = re.compile(r'[,\t\n\r\f\v]') + + +def parse_comma_separated_list(value, regexp=COMMA_SEPARATED_LIST_RE): + # type: (Union[Sequence[str], str]) -> List[str] + """Parse a comma-separated list. + + :param value: + String or list of strings to be parsed and normalized. + :param regexp: + Compiled regular expression used to split the value when it is a + string. + :type regexp: + _sre.SRE_Pattern + :returns: + List of values with whitespace stripped. + :rtype: + list + """ + if not value: + return [] + + if not isinstance(value, (list, tuple)): + value = regexp.split(value) + + item_gen = (item.strip() for item in value) + return [item for item in item_gen if item] + + +def normalize_paths(paths, parent=os.curdir): + # type: (Union[Sequence[str], str], str) -> List[str] + """Parse a comma-separated list of paths. + + :returns: + The normalized paths. + :rtype: + [str] + """ + return [normalize_path(p, parent) + for p in parse_comma_separated_list(paths)] + + +def normalize_path(path, parent=os.curdir): + # type: (str, str) -> str + """Normalize a single-path. + + :returns: + The normalized path. + :rtype: + str + """ + # NOTE(sigmavirus24): Using os.path.sep and os.path.altsep allow for + # Windows compatibility with both Windows-style paths (c:\\foo\bar) and + # Unix style paths (/foo/bar). + separator = os.path.sep + # NOTE(sigmavirus24): os.path.altsep may be None + alternate_separator = os.path.altsep or '' + if separator in path or (alternate_separator and + alternate_separator in path): + path = os.path.abspath(os.path.join(parent, path)) + return path.rstrip(separator + alternate_separator) + + +def _stdin_get_value_py3(): + stdin_value = sys.stdin.buffer.read() + fd = io.BytesIO(stdin_value) + try: + (coding, lines) = tokenize.detect_encoding(fd.readline) + return io.StringIO(stdin_value.decode(coding)) + except (LookupError, SyntaxError, UnicodeError): + return io.StringIO(stdin_value.decode('utf-8')) + + +def stdin_get_value(): + # type: () -> str + """Get and cache it so plugins can use it.""" + cached_value = getattr(stdin_get_value, 'cached_stdin', None) + if cached_value is None: + if sys.version_info < (3, 0): + stdin_value = io.BytesIO(sys.stdin.read()) + else: + stdin_value = _stdin_get_value_py3() + stdin_get_value.cached_stdin = stdin_value + cached_value = stdin_get_value.cached_stdin + return cached_value.getvalue() + + +def parse_unified_diff(diff=None): + # type: (str) -> List[str] + """Parse the unified diff passed on stdin. + + :returns: + dictionary mapping file names to sets of line numbers + :rtype: + dict + """ + # Allow us to not have to patch out stdin_get_value + if diff is None: + diff = stdin_get_value() + + number_of_rows = None + current_path = None + parsed_paths = collections.defaultdict(set) + for line in diff.splitlines(): + if number_of_rows: + # NOTE(sigmavirus24): Below we use a slice because stdin may be + # bytes instead of text on Python 3. + if line[:1] != '-': + number_of_rows -= 1 + # We're in the part of the diff that has lines starting with +, -, + # and ' ' to show context and the changes made. We skip these + # because the information we care about is the filename and the + # range within it. + # When number_of_rows reaches 0, we will once again start + # searching for filenames and ranges. + continue + + # NOTE(sigmavirus24): Diffs that we support look roughly like: + # diff a/file.py b/file.py + # ... + # --- a/file.py + # +++ b/file.py + # Below we're looking for that last line. Every diff tool that + # gives us this output may have additional information after + # ``b/file.py`` which it will separate with a \t, e.g., + # +++ b/file.py\t100644 + # Which is an example that has the new file permissions/mode. + # In this case we only care about the file name. + if line[:3] == '+++': + current_path = line[4:].split('\t', 1)[0] + # NOTE(sigmavirus24): This check is for diff output from git. + if current_path[:2] == 'b/': + current_path = current_path[2:] + # We don't need to do anything else. We have set up our local + # ``current_path`` variable. We can skip the rest of this loop. + # The next line we will see will give us the hung information + # which is in the next section of logic. + continue + + hunk_match = DIFF_HUNK_REGEXP.match(line) + # NOTE(sigmavirus24): pep8/pycodestyle check for: + # line[:3] == '@@ ' + # But the DIFF_HUNK_REGEXP enforces that the line start with that + # So we can more simply check for a match instead of slicing and + # comparing. + if hunk_match: + (row, number_of_rows) = [ + 1 if not group else int(group) + for group in hunk_match.groups() + ] + parsed_paths[current_path].update( + range(row, row + number_of_rows) + ) + + # We have now parsed our diff into a dictionary that looks like: + # {'file.py': set(range(10, 16), range(18, 20)), ...} + return parsed_paths + + +def is_windows(): + # type: () -> bool + """Determine if we're running on Windows. + + :returns: + True if running on Windows, otherwise False + :rtype: + bool + """ + return os.name == 'nt' + + +# NOTE(sigmavirus24): If and when https://bugs.python.org/issue27649 is fixed, +# re-enable multiprocessing support on Windows. +def can_run_multiprocessing_on_windows(): + # type: () -> bool + """Determine if we can use multiprocessing on Windows. + + This presently will **always** return False due to a `bug`_ in the + :mod:`multiprocessing` module on Windows. Once fixed, we will check + to ensure that the version of Python contains that fix (via version + inspection) and *conditionally* re-enable support on Windows. + + .. _bug: + https://bugs.python.org/issue27649 + + :returns: + True if the version of Python is modern enough, otherwise False + :rtype: + bool + """ + is_new_enough_python27 = (2, 7, 11) <= sys.version_info < (3, 0) + is_new_enough_python3 = sys.version_info > (3, 2) + return False and (is_new_enough_python27 or is_new_enough_python3) + + +def is_using_stdin(paths): + # type: (List[str]) -> bool + """Determine if we're going to read from stdin. + + :param list paths: + The paths that we're going to check. + :returns: + True if stdin (-) is in the path, otherwise False + :rtype: + bool + """ + return '-' in paths + + +def _default_predicate(*args): + return False + + +def filenames_from(arg, predicate=None): + # type: (str, callable) -> Generator + """Generate filenames from an argument. + + :param str arg: + Parameter from the command-line. + :param callable predicate: + Predicate to use to filter out filenames. If the predicate + returns ``True`` we will exclude the filename, otherwise we + will yield it. By default, we include every filename + generated. + :returns: + Generator of paths + """ + if predicate is None: + predicate = _default_predicate + + if predicate(arg): + return + + if os.path.isdir(arg): + for root, sub_directories, files in os.walk(arg): + if predicate(root): + sub_directories[:] = [] + continue + + # NOTE(sigmavirus24): os.walk() will skip a directory if you + # remove it from the list of sub-directories. + for directory in sub_directories: + joined = os.path.join(root, directory) + if predicate(joined): + sub_directories.remove(directory) + + for filename in files: + joined = os.path.join(root, filename) + if predicate(joined) or predicate(filename): + continue + yield joined + else: + yield arg + + +def fnmatch(filename, patterns, default=True): + # type: (str, List[str], bool) -> bool + """Wrap :func:`fnmatch.fnmatch` to add some functionality. + + :param str filename: + Name of the file we're trying to match. + :param list patterns: + Patterns we're using to try to match the filename. + :param bool default: + The default value if patterns is empty + :returns: + True if a pattern matches the filename, False if it doesn't. + ``default`` if patterns is empty. + """ + if not patterns: + return default + return any(_fnmatch.fnmatch(filename, pattern) for pattern in patterns) + + +def parameters_for(plugin): + # type: (flake8.plugins.manager.Plugin) -> Dict[str, bool] + """Return the parameters for the plugin. + + This will inspect the plugin and return either the function parameters + if the plugin is a function or the parameters for ``__init__`` after + ``self`` if the plugin is a class. + + :param plugin: + The internal plugin object. + :type plugin: + flake8.plugins.manager.Plugin + :returns: + A dictionary mapping the parameter name to whether or not it is + required (a.k.a., is positional only/does not have a default). + :rtype: + dict([(str, bool)]) + """ + func = plugin.plugin + is_class = not inspect.isfunction(func) + if is_class: # The plugin is a class + func = plugin.plugin.__init__ + + if sys.version_info < (3, 3): + argspec = inspect.getargspec(func) + start_of_optional_args = len(argspec[0]) - len(argspec[-1] or []) + parameter_names = argspec[0] + parameters = collections.OrderedDict([ + (name, position < start_of_optional_args) + for position, name in enumerate(parameter_names) + ]) + else: + parameters = collections.OrderedDict([ + (parameter.name, parameter.default is parameter.empty) + for parameter in inspect.signature(func).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD + ]) + + if is_class: + parameters.pop('self', None) + + return parameters + + +def get_python_version(): + """Find and format the python implementation and version. + + :returns: + Implementation name, version, and platform as a string. + :rtype: + str + """ + # The implementation isn't all that important. + try: + impl = platform.python_implementation() + " " + except AttributeError: # Python 2.5 + impl = '' + return '%s%s on %s' % (impl, platform.python_version(), platform.system()) diff --git a/contrib/flake8_debugger.py b/contrib/flake8_debugger.py old mode 100755 new mode 100644 diff --git a/contrib/flake8_import_order/__about__.py b/contrib/flake8_import_order/__about__.py old mode 100755 new mode 100644 index 91baecb..f263dd2 --- a/contrib/flake8_import_order/__about__.py +++ b/contrib/flake8_import_order/__about__.py @@ -1,32 +1,24 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. from __future__ import absolute_import, division, print_function __all__ = [ "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__email__", "__license__", "__copyright__", '__maintainer__', + '__maintainer_email__', ] __title__ = "flake8-import-order" __summary__ = ( "Flake8 and pylama plugin that checks the ordering of import statements." ) -__uri__ = "https://github.com/public/flake8-import-order" +__uri__ = "https://github.com/PyCQA/flake8-import-order" -__version__ = "0.6.1" +__version__ = "0.14.2" __author__ = "Alex Stapleton" __email__ = "alexs@prol.etari.at" +__maintainer__ = 'Phil Jones' +__maintainer_email__ = 'philip.graham.jones+flake8-import@gmail.com' + __license__ = "LGPLv3" -__copyright__ = "Copyright 2013-2015 %s" % __author__ +__copyright__ = "Copyright 2013-2016 %s" % __author__ diff --git a/contrib/flake8_import_order/__init__.py b/contrib/flake8_import_order/__init__.py old mode 100755 new mode 100644 index 5435d7f..8a5ab47 --- a/contrib/flake8_import_order/__init__.py +++ b/contrib/flake8_import_order/__init__.py @@ -1,14 +1,12 @@ import ast - -import pep8 +from collections import namedtuple from flake8_import_order.__about__ import ( __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ + __uri__, __version__, ) from flake8_import_order.stdlib_list import STDLIB_NAMES - __all__ = [ "__title__", "__summary__", "__uri__", "__version__", "__author__", "__email__", "__license__", "__copyright__", @@ -19,10 +17,19 @@ IMPORT_FUTURE = 0 IMPORT_STDLIB = 10 IMPORT_3RD_PARTY = 20 -IMPORT_APP = 30 -IMPORT_APP_RELATIVE = 40 +IMPORT_APP_PACKAGE = 30 +IMPORT_APP = 40 +IMPORT_APP_RELATIVE = 50 IMPORT_MIXED = -1 +ClassifiedImport = namedtuple( + 'ClassifiedImport', + [ + 'type', 'is_from', 'modules', 'names', 'start_line', 'end_line', + 'level', 'package', + ], +) + def root_package_name(name): p = ast.parse(name) @@ -33,297 +40,56 @@ def root_package_name(name): return None -def is_sorted(seq): - return sorted(seq) == list(seq) - - -def lower_strings(l): - if l is None: - return None - else: - return [e.lower() if hasattr(e, 'lower') else e for e in l] - - -def cmp_values(n, style): - if n[0] in (IMPORT_STDLIB, IMPORT_APP_RELATIVE) or style == "google": - return [ - n[0], - n[1], - lower_strings(n[2]), - n[3], - [lower_strings(x) for x in n[4]] - ] - else: - return [ - n[0], - lower_strings(n[1]), - n[2], - n[3], - [lower_strings(x) for x in n[4]] - ] - - class ImportVisitor(ast.NodeVisitor): - """ - This class visits all the import nodes at the root of tree and generates - sort keys for each import node. - - In practice this means that they are sorted according to something like - this tuple. - (stdlib, site_packages, names) - """ - - def __init__(self, filename, options): - self.filename = filename - self.options = options or {} + def __init__(self, application_import_names, application_package_names): self.imports = [] - - self.application_import_names = set( - self.options.get("application_import_names", []) - ) - self.style = self.options['import_order_style'] + self.application_import_names = application_import_names + self.application_package_names = application_package_names def visit_Import(self, node): # noqa - if node.col_offset != 0: - return - else: - self.imports.append(node) - return + if node.col_offset == 0: + modules = [alias.name for alias in node.names] + types_ = {self._classify_type(module) for module in modules} + if len(types_) == 1: + type_ = types_.pop() + else: + type_ = IMPORT_MIXED + classified_import = ClassifiedImport( + type_, False, modules, [], node.first_token.start[0], + node.last_token.end[0], 0, + root_package_name(modules[0]), + ) + self.imports.append(classified_import) def visit_ImportFrom(self, node): # noqa - if node.col_offset != 0: - return - else: - self.imports.append(node) - return - - def node_sort_key(self, node): - """ - Return a key that will sort the nodes in the correct - order for the Google Code Style guidelines. - """ - - if isinstance(node, ast.Import): - names = [nm.name for nm in node.names] - elif isinstance(node, ast.ImportFrom): - names = [node.module or ''] - else: - raise TypeError(type(node)) - - import_type = self._import_type(node, names[0]) - for name in names[1:]: - name_type = self._import_type(node, name) - if import_type != name_type: - import_type = IMPORT_MIXED - break - - imported_names = [ - [nm.name if nm.name != "*" - else "{0}.*".format(node.module), nm.asname] - for nm in node.names - ] - - if self.style == "google": - true_from_level = getattr(node, "level", -1) - - if true_from_level == -1: - from_level = 0 - is_not_star_import = False + if node.col_offset == 0: + module = node.module or '' + if node.level > 0: + type_ = IMPORT_APP_RELATIVE else: - from_level = true_from_level - is_not_star_import = ( - not any(nm.endswith("*") - for nm, asnm in imported_names) - ) - - else: - from_level = getattr(node, "level", -1) - is_not_star_import = ( - not any(nm.endswith("*") - for nm, asnm in imported_names) + type_ = self._classify_type(module) + names = [alias.name for alias in node.names] + classified_import = ClassifiedImport( + type_, True, [module], names, + node.first_token.start[0], node.last_token.end[0], + node.level, + root_package_name(module), ) + self.imports.append(classified_import) - n = ( - import_type, - names, - from_level, - is_not_star_import, - imported_names, - ) - - if n[0] == IMPORT_FUTURE: - group = (n[0], None, None, None, n[4]) - elif ( - n[0] in (IMPORT_STDLIB, IMPORT_APP_RELATIVE) or - self.style == 'google' - ): - group = (n[0], n[2], n[1], n[3], n[4]) - elif n[0] == IMPORT_3RD_PARTY: - group = (n[0], n[1], n[2], n[3], n[4]) - else: - group = n - - return group, n - - def _import_type(self, node, name): - if isinstance(name, int): - return None - - if name is None: - # relative import - return IMPORT_APP - - pkg = root_package_name(name) + def _classify_type(self, module): + package = root_package_name(module) - # Entirely not confusingly we use "False" for "True" in the flags. - - if pkg == "__future__": + if package == "__future__": return IMPORT_FUTURE - - elif pkg in self.application_import_names: + elif package in self.application_import_names: return IMPORT_APP - - elif isinstance(node, ast.ImportFrom) and node.level > 0: - return IMPORT_APP_RELATIVE - - elif pkg in STDLIB_NAMES: + elif package in self.application_package_names: + return IMPORT_APP_PACKAGE + elif package in STDLIB_NAMES: return IMPORT_STDLIB - else: # Not future, stdlib or an application import. # Must be 3rd party. return IMPORT_3RD_PARTY - - -class ImportOrderChecker(object): - visitor_class = ImportVisitor - options = None - - def __init__(self, filename, tree): - self.tree = tree - self.filename = filename - self.lines = None - - def load_file(self): - if self.filename in ("stdin", "-", None): - self.filename = "stdin" - self.lines = pep8.stdin_get_value().splitlines(True) - else: - self.lines = pep8.readlines(self.filename) - - if not self.tree: - self.tree = ast.parse("".join(self.lines)) - - def error(self, node, code, message): - raise NotImplemented() - - def check_order(self): - if not self.tree or not self.lines: - self.load_file() - - visitor = self.visitor_class(self.filename, self.options) - visitor.visit(self.tree) - - style = self.options['import_order_style'] - - prev_node = None - for node in visitor.imports: - # Lines with the noqa flag are ignored entirely - if pep8.noqa(self.lines[node.lineno - 1]): - continue - - n, k = visitor.node_sort_key(node) - - if style == "google": - cmp_n = cmp_values(n, style) - else: - cmp_n = n - - if cmp_n[-1] and not is_sorted(cmp_n[-1]): - sort_key = lambda s: s[0] - if style == "google": - sort_key = lambda s: s[0].lower() - should_be = ", ".join( - name[0] for name in - sorted(n[-1], key=sort_key)) - yield self.error( - node, "I101", - ( - "Imported names are in the wrong order. " - "Should be {0}".format(should_be) - ) - ) - - if prev_node is None: - prev_node = node - continue - - pn, pk = visitor.node_sort_key(prev_node) - - if style == "google": - cmp_pn = cmp_values(pn, style) - else: - cmp_pn = pn - - # FUTURES - # STDLIBS, STDLIB_FROMS - # 3RDPARTY[n], 3RDPARTY_FROM[n] - # 3RDPARTY[n+1], 3RDPARTY_FROM[n+1] - # APPLICATION, APPLICATION_FROM - - # import_type, names, level, is_star_import, imported_names, - - if n[0] == IMPORT_MIXED: - yield self.error( - node, "I666", - "Import statement mixes groups" - ) - prev_node = node - continue - - if cmp_n < cmp_pn: - def build_str(key): - level = key[2] - if level >= 0: - start = "from " + level * '.' - else: - start = "import " - return start + ", ".join(key[1]) - - first_str = build_str(k) - second_str = build_str(pk) - - yield self.error( - node, "I100", - ( - "Imports statements are in the wrong order. " - "{0} should be before {1}".format( - first_str, - second_str - ) - ) - ) - - lines_apart = node.lineno - prev_node.lineno - - is_app = ( - set([cmp_n[0], cmp_pn[0]]) != - set([IMPORT_APP, IMPORT_APP_RELATIVE]) - ) - - if lines_apart == 1 and (( - cmp_n[0] != cmp_pn[0] and - (style != "google" or is_app) - ) or ( - n[0] == IMPORT_3RD_PARTY and - style != 'google' and - root_package_name(cmp_n[1][0]) != - root_package_name(cmp_pn[1][0]) - )): - yield self.error( - node, "I201", - "Missing newline before sections or imports." - ) - - prev_node = node diff --git a/contrib/flake8_import_order/checker.py b/contrib/flake8_import_order/checker.py new file mode 100644 index 0000000..328fd6f --- /dev/null +++ b/contrib/flake8_import_order/checker.py @@ -0,0 +1,69 @@ +import ast + +import asttokens + +import pycodestyle + +from flake8_import_order import ImportVisitor +from flake8_import_order.styles import lookup_entry_point + +DEFAULT_IMPORT_ORDER_STYLE = 'cryptography' + + +class ImportOrderChecker(object): + visitor_class = ImportVisitor + options = None + + def __init__(self, filename, tree): + self.ast_tree = tree + self.filename = filename + self.lines = None + + def load_file(self): + if self.filename in ("stdin", "-", None): + self.filename = "stdin" + self.lines = pycodestyle.stdin_get_value().splitlines(True) + else: + self.lines = pycodestyle.readlines(self.filename) + + if self.ast_tree is None: + self.ast_tree = ast.parse(''.join(self.lines)) + + def error(self, error): + return error + + def check_order(self): + if not self.ast_tree or not self.lines: + self.load_file() + + tree = asttokens.ASTTokens( + ''.join(self.lines), parse=False, tree=self.ast_tree, + ).tree + + try: + style_entry_point = self.options['import_order_style'] + except KeyError: + style_entry_point = lookup_entry_point(DEFAULT_IMPORT_ORDER_STYLE) + style_cls = style_entry_point.load() + + if style_cls.accepts_application_package_names: + visitor = self.visitor_class( + self.options.get('application_import_names', []), + self.options.get('application_package_names', []), + ) + else: + visitor = self.visitor_class( + self.options.get('application_import_names', []), + [], + ) + visitor.visit(tree) + + imports = [] + for import_ in visitor.imports: + if not pycodestyle.noqa(self.lines[import_.start_line - 1]): + imports.append(import_) + + style = style_cls(imports) + + for error in style.check(): + yield self.error(error) diff --git a/contrib/flake8_import_order/flake8_linter.py b/contrib/flake8_import_order/flake8_linter.py old mode 100755 new mode 100644 index d3c0eab..c6fb1ba --- a/contrib/flake8_import_order/flake8_linter.py +++ b/contrib/flake8_import_order/flake8_linter.py @@ -1,52 +1,106 @@ from __future__ import absolute_import -import flake8_import_order -from flake8_import_order import DEFAULT_IMPORT_ORDER_STYLE, ImportOrderChecker +import optparse + +from flake8_import_order import __version__ +from flake8_import_order.checker import ( + DEFAULT_IMPORT_ORDER_STYLE, ImportOrderChecker, +) +from flake8_import_order.styles import list_entry_points, lookup_entry_point class Linter(ImportOrderChecker): name = "import-order" - version = flake8_import_order.__version__ + version = __version__ - def __init__(self, tree, filename): + def __init__(self, tree, filename, lines=None): super(Linter, self).__init__(filename, tree) + self.lines = lines @classmethod def add_options(cls, parser): # List of application import names. They go last. - parser.add_option( + register_opt( + parser, "--application-import-names", default="", action="store", type="string", - help="Import names to consider as application specific" + help="Import names to consider as application-specific", + parse_from_config=True, + comma_separated_list=True, + ) + register_opt( + parser, + "--application-package-names", + default="", + action="store", + type="string", + help=("Package names to consider as company-specific " + "(used only by 'appnexus' style)"), + parse_from_config=True, + comma_separated_list=True, ) - parser.add_option( + register_opt( + parser, "--import-order-style", default=DEFAULT_IMPORT_ORDER_STYLE, action="store", type="string", - help="Style to follow. Available: cryptography, google" + help=("Style to follow. Available: " + + ", ".join(cls.list_available_styles())), + parse_from_config=True, ) - parser.config_options.append("application-import-names") - parser.config_options.append("import-order-style") + + @staticmethod + def list_available_styles(): + entry_points = list_entry_points() + return sorted(entry_point.name for entry_point in entry_points) @classmethod def parse_options(cls, options): optdict = {} - names = options.application_import_names.split(",") + names = options.application_import_names + if not isinstance(names, list): + names = options.application_import_names.split(",") + + pkg_names = options.application_package_names + if not isinstance(pkg_names, list): + pkg_names = options.application_package_names.split(",") + + style_entry_point = lookup_entry_point(options.import_order_style) + optdict = dict( application_import_names=[n.strip() for n in names], - import_order_style=options.import_order_style, + application_package_names=[p.strip() for p in pkg_names], + import_order_style=style_entry_point, ) cls.options = optdict - def error(self, node, code, message): - lineno, col_offset = node.lineno, node.col_offset - return (lineno, col_offset, '{0} {1}'.format(code, message), Linter) + def error(self, error): + return ( + error.lineno, + 0, + "{0} {1}".format(error.code, error.message), + Linter, + ) def run(self): for error in self.check_order(): yield error + + +def register_opt(parser, *args, **kwargs): + try: + # Flake8 3.x registration + parser.add_option(*args, **kwargs) + except (optparse.OptionError, TypeError): + # Flake8 2.x registration + parse_from_config = kwargs.pop('parse_from_config', False) + kwargs.pop('comma_separated_list', False) + kwargs.pop('normalize_paths', False) + parser.add_option(*args, **kwargs) + if parse_from_config: + parser.config_options.append(args[-1].lstrip('-')) diff --git a/contrib/flake8_import_order/pylama_linter.py b/contrib/flake8_import_order/pylama_linter.py old mode 100755 new mode 100644 index 286468c..037eee1 --- a/contrib/flake8_import_order/pylama_linter.py +++ b/contrib/flake8_import_order/pylama_linter.py @@ -2,12 +2,16 @@ from pylama.lint import Linter as BaseLinter -from flake8_import_order import DEFAULT_IMPORT_ORDER_STYLE, ImportOrderChecker +from flake8_import_order import __version__ +from flake8_import_order.checker import ( + DEFAULT_IMPORT_ORDER_STYLE, ImportOrderChecker, +) +from flake8_import_order.styles import lookup_entry_point class Linter(ImportOrderChecker, BaseLinter): name = "import-order" - version = "0.1" + version = __version__ def __init__(self): super(Linter, self).__init__(None, None) @@ -15,21 +19,22 @@ def __init__(self): def allow(self, path): return path.endswith(".py") - def error(self, node, code, message): - lineno, col_offset = node.lineno, node.col_offset + def error(self, error): return { - "lnum": lineno, - "col": col_offset, - "text": message, - "type": code + 'lnum': error.lineno, + 'col': 0, + 'text': error.message, + 'type': error.code, } def run(self, path, **meta): self.filename = path - self.tree = None - self.options = dict( - {'import_order_style': DEFAULT_IMPORT_ORDER_STYLE}, - **meta) + self.ast_tree = None + meta.setdefault('import_order_style', DEFAULT_IMPORT_ORDER_STYLE) + meta['import_order_style'] = lookup_entry_point( + meta['import_order_style'] + ) + self.options = meta for error in self.check_order(): yield error diff --git a/contrib/flake8_import_order/stdlib_list.py b/contrib/flake8_import_order/stdlib_list.py old mode 100755 new mode 100644 index 3724ada..60570a8 --- a/contrib/flake8_import_order/stdlib_list.py +++ b/contrib/flake8_import_order/stdlib_list.py @@ -65,6 +65,7 @@ "builtins", "bz2", "cPickle", + "cProfile", "cStringIO", "calendar", "cd", @@ -81,6 +82,7 @@ "colorsys", "commands", "compileall", + "concurrent", "concurrent.futures", "configparser", "contextlib", @@ -197,12 +199,14 @@ "new", "nis", "nntplib", + "ntpath", "nturl2path", "numbers", "operator", "optparse", "os", "os.path", + "os2emxpath", "ossaudiodev", "parser", "pathlib", @@ -219,6 +223,8 @@ "posixfile", "posixpath", "pprint", + "profile", + "pstats", "pty", "pwd", "py_compile", @@ -238,6 +244,7 @@ "robotparser", "runpy", "sched", + "secrets", "select", "sets", "sgmllib", @@ -295,6 +302,7 @@ "tty", "turtle", "types", + "typing", "unicodedata", "unittest", "unittest.mock", diff --git a/contrib/flake8_import_order/styles.py b/contrib/flake8_import_order/styles.py new file mode 100644 index 0000000..9874ac3 --- /dev/null +++ b/contrib/flake8_import_order/styles.py @@ -0,0 +1,181 @@ +from collections import namedtuple + +from pkg_resources import iter_entry_points + +from flake8_import_order import ( + IMPORT_3RD_PARTY, IMPORT_APP, IMPORT_APP_RELATIVE, +) + +Error = namedtuple('Error', ['lineno', 'code', 'message']) + + +def list_entry_points(): + return iter_entry_points('flake8_import_order.styles') + + +def lookup_entry_point(name): + try: + return next(iter_entry_points('flake8_import_order.styles', name=name)) + except StopIteration: + raise LookupError('Unknown style {}'.format(name)) + + +class Style(object): + + accepts_application_package_names = False + + def __init__(self, imports): + self.imports = imports + + def check(self): + previous = None + for current in self.imports: + if current.type == -1: + yield Error( + current.start_line, + 'I666', + 'Import statement mixes groups', + ) + + correct_names = self.sorted_names(current.names) + if correct_names != current.names: + corrected = ', '.join(correct_names) + yield Error( + current.start_line, + 'I101', + "Imported names are in the wrong order. " + "Should be {0}".format(corrected), + ) + + if previous is not None: + if self.import_key(previous) > self.import_key(current): + first = self._explain(current) + second = self._explain(previous) + if first == second: + first = ", ".join(current.names) + second = ", ".join(previous.names) + yield Error( + current.start_line, + 'I100', + "Import statements are in the wrong order. " + "{0} should be before {1}".format(first, second), + ) + + spacing = current.start_line - previous.end_line + same_section = self.same_section(previous, current) + if not same_section and spacing == 1: + yield Error( + current.start_line, + 'I201', + 'Missing newline before sections or imports.', + ) + + if same_section and spacing > 1: + yield Error( + current.start_line, + 'I202', + 'Additional newline in a section of imports.', + ) + + previous = current + + @staticmethod + def sorted_names(names): + return names + + @staticmethod + def import_key(import_): + return (import_.type,) + + @staticmethod + def same_section(previous, current): + same_type = current.type == previous.type + both_first = ( + {previous.type, current.type} <= {IMPORT_APP, IMPORT_APP_RELATIVE} + ) + return same_type or both_first + + @staticmethod + def _explain(import_): + if import_.is_from: + text = 'from ' + import_.level * '.' + else: + text = 'import ' + return text + ', '.join(import_.modules) + + +class PEP8(Style): + pass + + +class Google(Style): + + @staticmethod + def sorted_names(names): + return sorted(names, key=Google.name_key) + + @staticmethod + def name_key(name): + return (name.lower(), name) + + @staticmethod + def import_key(import_): + modules = [Google.name_key(module) for module in import_.modules] + names = [Google.name_key(name) for name in import_.names] + return (import_.type, import_.level, modules, names) + + +class AppNexus(Google): + accepts_application_package_names = True + + +class Smarkets(Style): + + @staticmethod + def sorted_names(names): + return sorted(names, key=Smarkets.name_key) + + @staticmethod + def name_key(name): + return (name.lower(), name) + + @staticmethod + def import_key(import_): + modules = [Smarkets.name_key(module) for module in import_.modules] + names = [Smarkets.name_key(name) for name in import_.names] + return (import_.type, import_.is_from, import_.level, modules, names) + + +class Edited(Smarkets): + accepts_application_package_names = True + + +class Cryptography(Style): + + @staticmethod + def sorted_names(names): + return sorted(names) + + @staticmethod + def import_key(import_): + if import_.type in {IMPORT_3RD_PARTY, IMPORT_APP}: + return ( + import_.type, import_.package, import_.is_from, + import_.level, import_.modules, import_.names, + ) + else: + return ( + import_.type, '', import_.is_from, import_.level, + import_.modules, import_.names, + ) + + @staticmethod + def same_section(previous, current): + app_or_third = current.type in {IMPORT_3RD_PARTY, IMPORT_APP} + same_type = current.type == previous.type + both_relative = previous.type == current.type == IMPORT_APP_RELATIVE + same_package = previous.package == current.package + return ( + (not app_or_third and same_type or both_relative) or + (app_or_third and same_package) + ) diff --git a/contrib/mccabe.py b/contrib/mccabe.py index a86d208..c0cda75 100644 --- a/contrib/mccabe.py +++ b/contrib/mccabe.py @@ -7,6 +7,8 @@ import optparse import sys +import tokenize + from collections import defaultdict try: import ast @@ -14,7 +16,7 @@ except ImportError: # Python 2.5 from flake8.util import ast, iter_child_nodes -__version__ = '0.4.0' +__version__ = '0.6.1' class ASTVisitor(object): @@ -59,10 +61,11 @@ def dot_id(self): class PathGraph(object): - def __init__(self, name, entity, lineno): + def __init__(self, name, entity, lineno, column=0): self.name = name self.entity = entity self.lineno = lineno + self.column = column self.nodes = defaultdict(list) def connect(self, n1, n2): @@ -114,7 +117,7 @@ def visitFunctionDef(self, node): else: entity = node.name - name = '%d:1: %r' % (node.lineno, entity) + name = '%d:%d: %r' % (node.lineno, node.col_offset, entity) if self.graph is not None: # closure @@ -126,7 +129,7 @@ def visitFunctionDef(self, node): self.graph.connect(pathnode, bottom) self.tail = bottom else: - self.graph = PathGraph(name, entity, node.lineno) + self.graph = PathGraph(name, entity, node.lineno, node.col_offset) pathnode = PathNode(name) self.tail = pathnode self.dispatch_list(node.body) @@ -157,10 +160,11 @@ def visitSimpleStatement(self, node): name = "Stmt %d" % lineno self.appendPathNode(name) - visitAssert = visitAssign = visitAugAssign = visitDelete = visitPrint = \ - visitRaise = visitYield = visitImport = visitCall = visitSubscript = \ - visitPass = visitContinue = visitBreak = visitGlobal = visitReturn = \ - visitAwait = visitSimpleStatement + def default(self, node, *args): + if isinstance(node, ast.stmt): + self.visitSimpleStatement(node) + else: + super(PathGraphingAstVisitor, self).default(node, *args) def visitLoop(self, node): name = "Loop %d" % node.lineno @@ -176,7 +180,7 @@ def _subgraph(self, node, name, extra_blocks=()): """create the subgraphs representing any `if` and `for` statements""" if self.graph is None: # global loop - self.graph = PathGraph(name, name, node.lineno) + self.graph = PathGraph(name, name, node.lineno, node.col_offset) pathnode = PathNode(name) self._subgraph_parse(node, pathnode, extra_blocks) self.graphs["%s%s" % (self.classname, name)] = self.graph @@ -227,16 +231,29 @@ class McCabeChecker(object): version = __version__ _code = 'C901' _error_tmpl = "C901 %r is too complex (%d)" - max_complexity = 0 + max_complexity = -1 def __init__(self, tree, filename): self.tree = tree @classmethod def add_options(cls, parser): - parser.add_option('--max-complexity', default=-1, action='store', - type='int', help="McCabe complexity threshold") - parser.config_options.append('max-complexity') + flag = '--max-complexity' + kwargs = { + 'default': -1, + 'action': 'store', + 'type': 'int', + 'help': 'McCabe complexity threshold', + 'parse_from_config': 'True', + } + config_opts = getattr(parser, 'config_options', None) + if isinstance(config_opts, list): + # Flake8 2.x + kwargs.pop('parse_from_config') + parser.add_option(flag, **kwargs) + parser.config_options.append('max-complexity') + else: + parser.add_option(flag, **kwargs) @classmethod def parse_options(cls, options): @@ -250,7 +267,7 @@ def run(self): for graph in visitor.graphs.values(): if graph.complexity() > self.max_complexity: text = self._error_tmpl % (graph.entity, graph.complexity()) - yield graph.lineno, 0, text, type(self) + yield graph.lineno, graph.column, text, type(self) def get_code_complexity(code, threshold=7, filename='stdin'): @@ -279,6 +296,23 @@ def get_module_complexity(module_path, threshold=7): return get_code_complexity(code, threshold, filename=module_path) +def _read(filename): + if (2, 5) < sys.version_info < (3, 0): + with open(filename, 'rU') as f: + return f.read() + elif (3, 0) <= sys.version_info < (4, 0): + """Read the source code.""" + try: + with open(filename, 'rb') as f: + (encoding, _) = tokenize.detect_encoding(f.readline) + except (LookupError, SyntaxError, UnicodeError): + # Fall back if file encoding is improperly declared + with open(filename, encoding='latin-1') as f: + return f.read() + with open(filename, 'r', encoding=encoding) as f: + return f.read() + + def main(argv=None): if argv is None: argv = sys.argv[1:] @@ -291,8 +325,7 @@ def main(argv=None): options, args = opar.parse_args(argv) - with open(args[0], "rU") as mod: - code = mod.read() + code = _read(args[0]) tree = compile(code, args[0], "exec", ast.PyCF_ONLY_AST) visitor = PathGraphingAstVisitor() visitor.preorder(tree, visitor) diff --git a/contrib/pep8.py b/contrib/pycodestyle.py old mode 100644 new mode 100755 similarity index 87% rename from contrib/pep8.py rename to contrib/pycodestyle.py index 3c950d4..88f870f --- a/contrib/pep8.py +++ b/contrib/pycodestyle.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -# pep8.py - Check Python source code formatting, according to PEP 8 +# pycodestyle.py - Check Python source code formatting, according to PEP 8 +# # Copyright (C) 2006-2009 Johann C. Rocholl # Copyright (C) 2009-2014 Florent Xicluna # Copyright (C) 2014-2016 Ian Lee @@ -28,10 +29,10 @@ Check Python source code formatting, according to PEP 8. For usage and a list of options, try this: -$ python pep8.py -h +$ python pycodestyle.py -h This program and its regression test suite live here: -https://github.com/pycqa/pep8 +https://github.com/pycqa/pycodestyle Groups of errors and warnings: E errors @@ -47,37 +48,51 @@ """ from __future__ import with_statement +import inspect +import keyword import os -import sys import re +import sys import time -import inspect -import keyword import tokenize -from optparse import OptionParser +import warnings +import bisect + +try: + from functools import lru_cache +except ImportError: + def lru_cache(maxsize=128): # noqa as it's a fake implementation. + """Does not really need a real a lru_cache, it's just optimization, so + let's just do nothing here. Python 3.2+ will just get better + performances, time to upgrade? + """ + return lambda function: function + from fnmatch import fnmatch +from optparse import OptionParser + try: from configparser import RawConfigParser from io import TextIOWrapper except ImportError: from ConfigParser import RawConfigParser -__version__ = '1.7.0' +__version__ = '2.3.1' DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' -DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704' +DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704,W503' try: if sys.platform == 'win32': - USER_CONFIG = os.path.expanduser(r'~\.pep8') + USER_CONFIG = os.path.expanduser(r'~\.pycodestyle') else: USER_CONFIG = os.path.join( os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), - 'pep8' + 'pycodestyle' ) except ImportError: USER_CONFIG = None -PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8') +PROJECT_CONFIG = ('setup.cfg', 'tox.ini') TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') MAX_LINE_LENGTH = 79 REPORT_FORMAT = { @@ -117,17 +132,63 @@ OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') LAMBDA_REGEX = re.compile(r'\blambda\b') HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') +STARTSWITH_DEF_REGEX = re.compile(r'^(async\s+def|def)\b') +STARTSWITH_TOP_LEVEL_REGEX = re.compile(r'^(async\s+def\s+|def\s+|class\s+|@)') +STARTSWITH_INDENT_STATEMENT_REGEX = re.compile( + r'^\s*({0})\b'.format('|'.join(s.replace(' ', r'\s+') for s in ( + 'def', 'async def', + 'for', 'async for', + 'if', 'elif', 'else', + 'try', 'except', 'finally', + 'with', 'async with', + 'class', + 'while', + ))) +) +DUNDER_REGEX = re.compile(r'^__([^\s]+)__ = ') # Work around Python < 2.6 behaviour, which does not generate NL after # a comment which is on a line by itself. COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' +_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} + + +def _get_parameters(function): + if sys.version_info >= (3, 3): + return [parameter.name + for parameter + in inspect.signature(function).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] + else: + return inspect.getargspec(function)[0] + + +def register_check(check, codes=None): + """Register a new check object.""" + def _add_check(check, kind, codes, args): + if check in _checks[kind]: + _checks[kind][check][0].extend(codes or []) + else: + _checks[kind][check] = (codes or [''], args) + if inspect.isfunction(check): + args = _get_parameters(check) + if args and args[0] in ('physical_line', 'logical_line'): + if codes is None: + codes = ERRORCODE_REGEX.findall(check.__doc__ or '') + _add_check(check, args[0], codes, args) + elif inspect.isclass(check): + if _get_parameters(check.__init__)[:2] == ['self', 'tree']: + _add_check(check, 'tree', codes, None) + return check + + ############################################################################## # Plugins (check functions) for physical lines ############################################################################## - +@register_check def tabs_or_spaces(physical_line, indent_char): r"""Never mix tabs and spaces. @@ -147,6 +208,7 @@ def tabs_or_spaces(physical_line, indent_char): return offset, "E101 indentation contains mixed spaces and tabs" +@register_check def tabs_obsolete(physical_line): r"""For new projects, spaces-only are strongly recommended over tabs. @@ -158,6 +220,7 @@ def tabs_obsolete(physical_line): return indent.index('\t'), "W191 indentation contains tabs" +@register_check def trailing_whitespace(physical_line): r"""Trailing whitespace is superfluous. @@ -179,6 +242,7 @@ def trailing_whitespace(physical_line): return 0, "W293 blank line contains whitespace" +@register_check def trailing_blank_lines(physical_line, lines, line_number, total_lines): r"""Trailing blank lines are superfluous. @@ -195,7 +259,8 @@ def trailing_blank_lines(physical_line, lines, line_number, total_lines): return len(physical_line), "W292 no newline at end of file" -def maximum_line_length(physical_line, max_line_length, multiline): +@register_check +def maximum_line_length(physical_line, max_line_length, multiline, noqa): r"""Limit all lines to a maximum of 79 characters. There are still many devices around that are limited to 80 character @@ -209,7 +274,7 @@ def maximum_line_length(physical_line, max_line_length, multiline): """ line = physical_line.rstrip() length = len(line) - if length > max_line_length and not noqa(line): + if length > max_line_length and not noqa: # Special case for long URLs in multi-line docstrings or comments, # but still report the error when the 72 first chars are whitespaces. chunks = line.split() @@ -233,8 +298,11 @@ def maximum_line_length(physical_line, max_line_length, multiline): ############################################################################## +@register_check def blank_lines(logical_line, blank_lines, indent_level, line_number, - blank_before, previous_logical, previous_indent_level): + blank_before, previous_logical, + previous_unindented_logical_line, previous_indent_level, + lines): r"""Separate top-level function and class definitions with two blank lines. Method definitions inside a class are separated by a single blank line. @@ -246,13 +314,19 @@ def blank_lines(logical_line, blank_lines, indent_level, line_number, Use blank lines in functions, sparingly, to indicate logical sections. Okay: def a():\n pass\n\n\ndef b():\n pass + Okay: def a():\n pass\n\n\nasync def b():\n pass Okay: def a():\n pass\n\n\n# Foo\n# Bar\n\ndef b():\n pass + Okay: default = 1\nfoo = 1 + Okay: classify = 1\nfoo = 1 E301: class Foo:\n b = 0\n def bar():\n pass E302: def a():\n pass\n\ndef b(n):\n pass + E302: def a():\n pass\n\nasync def b(n):\n pass E303: def a():\n pass\n\n\n\ndef b(n):\n pass E303: def a():\n\n\n\n pass E304: @decorator\n\ndef a():\n pass + E305: def a():\n pass\na() + E306: def a():\n def b():\n pass\n def c():\n pass """ if line_number < 3 and not previous_logical: return # Don't expect blank lines before the first line @@ -261,15 +335,33 @@ def blank_lines(logical_line, blank_lines, indent_level, line_number, yield 0, "E304 blank lines found after function decorator" elif blank_lines > 2 or (indent_level and blank_lines == 2): yield 0, "E303 too many blank lines (%d)" % blank_lines - elif logical_line.startswith(('def ', 'class ', '@')): + elif STARTSWITH_TOP_LEVEL_REGEX.match(logical_line): if indent_level: if not (blank_before or previous_indent_level < indent_level or DOCSTRING_REGEX.match(previous_logical)): - yield 0, "E301 expected 1 blank line, found 0" + ancestor_level = indent_level + nested = False + # Search backwards for a def ancestor or tree root (top level). + for line in lines[line_number - 2::-1]: + if line.strip() and expand_indent(line) < ancestor_level: + ancestor_level = expand_indent(line) + nested = line.lstrip().startswith('def ') + if nested or ancestor_level == 0: + break + if nested: + yield 0, "E306 expected 1 blank line before a " \ + "nested definition, found 0" + else: + yield 0, "E301 expected 1 blank line, found 0" elif blank_before != 2: yield 0, "E302 expected 2 blank lines, found %d" % blank_before + elif (logical_line and not indent_level and blank_before != 2 and + previous_unindented_logical_line.startswith(('def ', 'class '))): + yield 0, "E305 expected 2 blank lines after " \ + "class or function definition, found %d" % blank_before +@register_check def extraneous_whitespace(logical_line): r"""Avoid extraneous whitespace. @@ -302,6 +394,7 @@ def extraneous_whitespace(logical_line): yield found, "%s whitespace before '%s'" % (code, char) +@register_check def whitespace_around_keywords(logical_line): r"""Avoid extraneous whitespace around keywords. @@ -325,6 +418,25 @@ def whitespace_around_keywords(logical_line): yield match.start(2), "E271 multiple spaces after keyword" +@register_check +def missing_whitespace_after_import_keyword(logical_line): + r"""Multiple imports in form from x import (a, b, c) should have space + between import statement and parenthesised name list. + + Okay: from foo import (bar, baz) + E275: from foo import(bar, baz) + E275: from importable.module import(bar, baz) + """ + line = logical_line + indicator = ' import(' + if line.startswith('from '): + found = line.find(indicator) + if -1 < found: + pos = found + len(indicator) - 1 + yield pos, "E275 missing whitespace after keyword" + + +@register_check def missing_whitespace(logical_line): r"""Each comma, semicolon or colon should be followed by whitespace. @@ -351,6 +463,7 @@ def missing_whitespace(logical_line): yield index, "E231 missing whitespace after '%s'" % char +@register_check def indentation(logical_line, previous_logical, indent_char, indent_level, previous_indent_level): r"""Use 4 spaces per indentation level. @@ -382,6 +495,7 @@ def indentation(logical_line, previous_logical, indent_char, yield 0, tmpl % (3 + c, "unexpected indentation") +@register_check def continued_indentation(logical_line, tokens, indent_level, hang_closing, indent_char, noqa, verbose): r"""Continuation lines indentation. @@ -565,7 +679,7 @@ def continued_indentation(logical_line, tokens, indent_level, hang_closing, break assert len(indent) == depth + 1 if start[1] not in indent_chances: - # allow to line up tokens + # allow lining up tokens indent_chances[start[1]] = text last_token_multiline = (start[0] != end[0]) @@ -581,6 +695,7 @@ def continued_indentation(logical_line, tokens, indent_level, hang_closing, yield pos, "%s with same indent as next logical line" % code +@register_check def whitespace_before_parameters(logical_line, tokens): r"""Avoid extraneous whitespace. @@ -613,6 +728,7 @@ def whitespace_before_parameters(logical_line, tokens): prev_end = end +@register_check def whitespace_around_operator(logical_line): r"""Avoid extraneous whitespace around an operator. @@ -636,6 +752,7 @@ def whitespace_around_operator(logical_line): yield match.start(2), "E222 multiple spaces after operator" +@register_check def missing_whitespace_around_operator(logical_line, tokens): r"""Surround operators with a single space on either side. @@ -728,6 +845,7 @@ def missing_whitespace_around_operator(logical_line, tokens): prev_end = end +@register_check def whitespace_around_comma(logical_line): r"""Avoid extraneous whitespace after a comma or a colon. @@ -746,6 +864,7 @@ def whitespace_around_comma(logical_line): yield found, "E241 multiple spaces after '%s'" % m.group()[0] +@register_check def whitespace_around_named_parameter_equals(logical_line, tokens): r"""Don't use spaces around the '=' sign in function arguments. @@ -759,6 +878,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): Okay: boolean(a <= b) Okay: boolean(a >= b) Okay: def foo(arg: int = 42): + Okay: async def foo(arg: int = 42): E251: def complex(real, imag = 0.0): E251: return magic(r = real, i = imag) @@ -767,7 +887,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): no_space = False prev_end = None annotated_func_arg = False - in_def = logical_line.startswith('def') + in_def = bool(STARTSWITH_DEF_REGEX.match(logical_line)) message = "E251 unexpected spaces around keyword / parameter equals" for token_type, text, start, end, line in tokens: if token_type == tokenize.NL: @@ -777,9 +897,9 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): if start != prev_end: yield (prev_end, message) if token_type == tokenize.OP: - if text == '(': + if text in '([': parens += 1 - elif text == ')': + elif text in ')]': parens -= 1 elif in_def and text == ':' and parens == 1: annotated_func_arg = True @@ -795,6 +915,7 @@ def whitespace_around_named_parameter_equals(logical_line, tokens): prev_end = end +@register_check def whitespace_before_comment(logical_line, tokens): r"""Separate inline comments by at least two spaces. @@ -836,8 +957,9 @@ def whitespace_before_comment(logical_line, tokens): prev_end = end +@register_check def imports_on_separate_lines(logical_line): - r"""Imports should usually be on separate lines. + r"""Place imports on separate lines. Okay: import os\nimport sys E401: import sys, os @@ -855,17 +977,22 @@ def imports_on_separate_lines(logical_line): yield found, "E401 multiple imports on one line" +@register_check def module_imports_on_top_of_file( logical_line, indent_level, checker_state, noqa): - r"""Imports are always put at the top of the file, just after any module - comments and docstrings, and before module globals and constants. + r"""Place imports at the top of the file. + + Always put imports at the top of the file, just after any module comments + and docstrings, and before module globals and constants. Okay: import os Okay: # this is a comment\nimport os Okay: '''this is a module docstring'''\nimport os Okay: r'''this is a module docstring'''\nimport os - Okay: try:\n import x\nexcept:\n pass\nelse:\n pass\nimport y - Okay: try:\n import x\nexcept:\n pass\nfinally:\n pass\nimport y + Okay: + try:\n\timport x\nexcept ImportError:\n\tpass\nelse:\n\tpass\nimport y + Okay: + try:\n\timport x\nexcept ImportError:\n\tpass\nfinally:\n\tpass\nimport y E402: a=1\nimport os E402: 'One string'\n"Two string"\nimport os E402: a=1\nfrom sys import x @@ -891,6 +1018,8 @@ def is_string_literal(line): if line.startswith('import ') or line.startswith('from '): if checker_state.get('seen_non_imports', False): yield 0, "E402 module level import not at top of file" + elif re.match(DUNDER_REGEX, line): + return elif any(line.startswith(kw) for kw in allowed_try_keywords): # Allow try, except, else, finally keywords intermixed with imports in # order to support conditional importing @@ -905,6 +1034,7 @@ def is_string_literal(line): checker_state['seen_non_imports'] = True +@register_check def compound_statements(logical_line): r"""Compound statements (on the same line) are generally discouraged. @@ -936,22 +1066,25 @@ def compound_statements(logical_line): line = logical_line last_char = len(line) - 1 found = line.find(':') + prev_found = 0 + counts = dict((char, 0) for char in '{}[]()') while -1 < found < last_char: - before = line[:found] - if ((before.count('{') <= before.count('}') and # {'a': 1} (dict) - before.count('[') <= before.count(']') and # [1:2] (slice) - before.count('(') <= before.count(')'))): # (annotation) - lambda_kw = LAMBDA_REGEX.search(before) + update_counts(line[prev_found:found], counts) + if ((counts['{'] <= counts['}'] and # {'a': 1} (dict) + counts['['] <= counts[']'] and # [1:2] (slice) + counts['('] <= counts[')'])): # (annotation) + lambda_kw = LAMBDA_REGEX.search(line, 0, found) if lambda_kw: before = line[:lambda_kw.start()].rstrip() if before[-1:] == '=' and isidentifier(before[:-1].strip()): yield 0, ("E731 do not assign a lambda expression, use a " "def") break - if before.startswith('def '): + if STARTSWITH_DEF_REGEX.match(line): yield 0, "E704 multiple statements on one line (def)" - else: + elif STARTSWITH_INDENT_STATEMENT_REGEX.match(line): yield found, "E701 multiple statements on one line (colon)" + prev_found = found found = line.find(':', found + 1) found = line.find(';') while -1 < found: @@ -962,6 +1095,7 @@ def compound_statements(logical_line): found = line.find(';', found + 1) +@register_check def explicit_line_join(logical_line, tokens): r"""Avoid explicit line join between brackets. @@ -1001,6 +1135,7 @@ def explicit_line_join(logical_line, tokens): parens -= 1 +@register_check def break_around_binary_operator(logical_line, tokens): r""" Avoid breaks before binary operators. @@ -1017,16 +1152,22 @@ def break_around_binary_operator(logical_line, tokens): Okay: x = '''\n''' + '' Okay: foo(x,\n -y) Okay: foo(x, # comment\n -y) + Okay: var = (1 &\n ~2) + Okay: var = (1 /\n -2) + Okay: var = (1 +\n -1 +\n -2) """ def is_binary_operator(token_type, text): # The % character is strictly speaking a binary operator, but the # common usage seems to be to put it next to the format parameters, # after a line break. return ((token_type == tokenize.OP or text in ['and', 'or']) and - text not in "()[]{},:.;@=%") + text not in "()[]{},:.;@=%~") line_break = False unary_context = True + # Previous non-newline token types and text + previous_token_type = None + previous_text = None for token_type, text, start, end, line in tokens: if token_type == tokenize.COMMENT: continue @@ -1034,12 +1175,17 @@ def is_binary_operator(token_type, text): line_break = True else: if (is_binary_operator(token_type, text) and line_break and - not unary_context): + not unary_context and + not is_binary_operator(previous_token_type, + previous_text)): yield start, "W503 line break before binary operator" unary_context = text in '([{,;' line_break = False + previous_token_type = token_type + previous_text = text +@register_check def comparison_to_singleton(logical_line, noqa): r"""Comparison to singletons should use "is" or "is not". @@ -1074,6 +1220,7 @@ def comparison_to_singleton(logical_line, noqa): (code, singleton, msg)) +@register_check def comparison_negative(logical_line): r"""Negative comparison should be done using "not in" and "is not". @@ -1095,6 +1242,7 @@ def comparison_negative(logical_line): yield pos, "E714 test for object identity should be 'is not'" +@register_check def comparison_type(logical_line, noqa): r"""Object type comparisons should always use isinstance(). @@ -1118,6 +1266,77 @@ def comparison_type(logical_line, noqa): yield match.start(), "E721 do not compare types, use 'isinstance()'" +@register_check +def bare_except(logical_line, noqa): + r"""When catching exceptions, mention specific exceptions when possible. + + Okay: except Exception: + Okay: except BaseException: + E722: except: + """ + if noqa: + return + + regex = re.compile(r"except\s*:") + match = regex.match(logical_line) + if match: + yield match.start(), "E722 do not use bare 'except'" + + +@register_check +def ambiguous_identifier(logical_line, tokens): + r"""Never use the characters 'l', 'O', or 'I' as variable names. + + In some fonts, these characters are indistinguishable from the numerals + one and zero. When tempted to use 'l', use 'L' instead. + + Okay: L = 0 + Okay: o = 123 + Okay: i = 42 + E741: l = 0 + E741: O = 123 + E741: I = 42 + + Variables can be bound in several other contexts, including class and + function definitions, 'global' and 'nonlocal' statements, exception + handlers, and 'with' statements. + + Okay: except AttributeError as o: + Okay: with lock as L: + E741: except AttributeError as O: + E741: with lock as l: + E741: global I + E741: nonlocal l + E742: class I(object): + E743: def l(x): + """ + idents_to_avoid = ('l', 'O', 'I') + prev_type, prev_text, prev_start, prev_end, __ = tokens[0] + for token_type, text, start, end, line in tokens[1:]: + ident = pos = None + # identifiers on the lhs of an assignment operator + if token_type == tokenize.OP and '=' in text: + if prev_text in idents_to_avoid: + ident = prev_text + pos = prev_start + # identifiers bound to a value with 'as', 'global', or 'nonlocal' + if prev_text in ('as', 'global', 'nonlocal'): + if text in idents_to_avoid: + ident = text + pos = start + if prev_text == 'class': + if text in idents_to_avoid: + yield start, "E742 ambiguous class definition '%s'" % text + if prev_text == 'def': + if text in idents_to_avoid: + yield start, "E743 ambiguous function definition '%s'" % text + if ident: + yield pos, "E741 ambiguous variable name '%s'" % ident + prev_text = text + prev_start = start + + +@register_check def python_3000_has_key(logical_line, noqa): r"""The {}.has_key() method is removed in Python 3: use the 'in' operator. @@ -1129,6 +1348,7 @@ def python_3000_has_key(logical_line, noqa): yield pos, "W601 .has_key() is deprecated, use 'in'" +@register_check def python_3000_raise_comma(logical_line): r"""When raising an exception, use "raise ValueError('message')". @@ -1142,6 +1362,7 @@ def python_3000_raise_comma(logical_line): yield match.end() - 1, "W602 deprecated form of raising exception" +@register_check def python_3000_not_equal(logical_line): r"""New code should always use != instead of <>. @@ -1155,8 +1376,9 @@ def python_3000_not_equal(logical_line): yield pos, "W603 '<>' is deprecated, use '!='" +@register_check def python_3000_backticks(logical_line): - r"""Backticks are removed in Python 3: use repr() instead. + r"""Use repr() instead of backticks in Python 3. Okay: val = repr(1 + 2) W604: val = `1 + 2` @@ -1166,6 +1388,57 @@ def python_3000_backticks(logical_line): yield pos, "W604 backticks are deprecated, use 'repr()'" +@register_check +def python_3000_invalid_escape_sequence(logical_line, tokens): + r"""Invalid escape sequences are deprecated in Python 3.6. + + Okay: regex = r'\.png$' + W605: regex = '\.png$' + """ + # https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals + valid = [ + '\n', + '\\', + '\'', + '"', + 'a', + 'b', + 'f', + 'n', + 'r', + 't', + 'v', + '0', '1', '2', '3', '4', '5', '6', '7', + 'x', + + # Escape sequences only recognized in string literals + 'N', + 'u', + 'U', + ] + + for token_type, text, start, end, line in tokens: + if token_type == tokenize.STRING: + quote = text[-3:] if text[-3:] in ('"""', "'''") else text[-1] + # Extract string modifiers (e.g. u or r) + quote_pos = text.index(quote) + prefix = text[:quote_pos].lower() + start = quote_pos + len(quote) + string = text[start:-len(quote)] + + if 'r' not in prefix: + pos = string.find('\\') + while pos >= 0: + pos += 1 + if string[pos] not in valid: + yield ( + pos, + "W605 invalid escape sequence '\\%s'" % + string[pos], + ) + pos = string.find('\\', pos + 1) + + ############################################################################## # Helper functions ############################################################################## @@ -1187,7 +1460,7 @@ def readlines(filename): with open(filename, 'rb') as f: (coding, lines) = tokenize.detect_encoding(f.readline) f = TextIOWrapper(f, coding, line_buffering=True) - return [l.decode(coding) for l in lines] + f.readlines() + return [line.decode(coding) for line in lines] + f.readlines() except (LookupError, SyntaxError, UnicodeError): # Fall back if file encoding is improperly declared with open(filename, encoding='latin-1') as f: @@ -1195,8 +1468,10 @@ def readlines(filename): isidentifier = str.isidentifier def stdin_get_value(): + """Read the value from stdin.""" return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() -noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search + +noqa = lru_cache(512)(re.compile(r'# no(?:qa|pep8)\b', re.I).search) def expand_indent(line): @@ -1299,8 +1574,18 @@ def filename_match(filename, patterns, default=True): return any(fnmatch(filename, pattern) for pattern in patterns) +def update_counts(s, counts): + r"""Adds one to the counts of each appearance of characters in s, + for characters in counts""" + for char in s: + if char in counts: + counts[char] += 1 + + def _is_eol_token(token): return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' + + if COMMENT_WITH_NL: def _is_eol_token(token, _eol_token=_is_eol_token): return _eol_token(token) or (token[0] == tokenize.COMMENT and @@ -1311,48 +1596,6 @@ def _is_eol_token(token, _eol_token=_is_eol_token): ############################################################################## -_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} - - -def _get_parameters(function): - if sys.version_info >= (3, 3): - return [parameter.name - for parameter - in inspect.signature(function).parameters.values() - if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] - else: - return inspect.getargspec(function)[0] - - -def register_check(check, codes=None): - """Register a new check object.""" - def _add_check(check, kind, codes, args): - if check in _checks[kind]: - _checks[kind][check][0].extend(codes or []) - else: - _checks[kind][check] = (codes or [''], args) - if inspect.isfunction(check): - args = _get_parameters(check) - if args and args[0] in ('physical_line', 'logical_line'): - if codes is None: - codes = ERRORCODE_REGEX.findall(check.__doc__ or '') - _add_check(check, args[0], codes, args) - elif inspect.isclass(check): - if _get_parameters(check.__init__)[:2] == ['self', 'tree']: - _add_check(check, 'tree', codes, None) - - -def init_checks_registry(): - """Register all globally visible functions. - - The first argument name is either 'physical_line' or 'logical_line'. - """ - mod = inspect.getmodule(register_check) - for (name, function) in inspect.getmembers(mod, inspect.isfunction): - register_check(function) -init_checks_registry() - - class Checker(object): """Load a Python source file, tokenize it, check coding style.""" @@ -1397,6 +1640,7 @@ def __init__(self, filename=None, lines=None, self.lines[0] = self.lines[0][3:] self.report = report or options.report self.report_error = self.report.error + self.noqa = False def report_invalid_syntax(self): """Check if the syntax is valid.""" @@ -1429,7 +1673,7 @@ def run_check(self, check, argument_names): return check(*arguments) def init_checker_state(self, name, argument_names): - """ Prepares a custom state for the specific checker plugin.""" + """Prepare custom state for the specific checker plugin.""" if 'checker_state' in argument_names: self.checker_state = self._checker_states.setdefault(name, {}) @@ -1482,10 +1726,10 @@ def check_logical(self): """Build a line from tokens and run all logical checks on it.""" self.report.increment_logical_line() mapping = self.build_tokens_line() - if not mapping: return + mapping_offsets = [offset for offset, _ in mapping] (start_row, start_col) = mapping[0][1] start_line = self.lines[start_row - 1] self.indent_level = expand_indent(start_line[:start_col]) @@ -1499,14 +1743,17 @@ def check_logical(self): self.init_checker_state(name, argument_names) for offset, text in self.run_check(check, argument_names) or (): if not isinstance(offset, tuple): - for token_offset, pos in mapping: - if offset <= token_offset: - break + # As mappings are ordered, bisecting is a fast way + # to find a given offset in them. + token_offset, pos = mapping[bisect.bisect_left( + mapping_offsets, offset)] offset = (pos[0], pos[1] + offset - token_offset) self.report_error(offset[0], offset[1], text, check) if self.logical_line: self.previous_indent_level = self.indent_level self.previous_logical = self.logical_line + if not self.indent_level: + self.previous_unindented_logical_line = self.logical_line self.blank_lines = 0 self.tokens = [] @@ -1531,6 +1778,7 @@ def generate_tokens(self): for token in tokengen: if token[2][0] > self.total_lines: return + self.noqa = token[4] and noqa(token[4]) self.maybe_check_physical(token) yield token except (SyntaxError, tokenize.TokenError): @@ -1561,7 +1809,9 @@ def maybe_check_physical(self, token): return self.multiline = True self.line_number = token[2][0] - for line in token[1].split('\n')[:-1]: + _, src, (_, offset), _, _ = token + src = self.lines[self.line_number - 1][:offset] + src + for line in src.split('\n')[:-1]: self.check_physical(line + '\n') self.line_number += 1 self.multiline = False @@ -1576,6 +1826,7 @@ def check_all(self, expected=None, line_offset=0): self.indent_char = None self.indent_level = self.previous_indent_level = 0 self.previous_logical = '' + self.previous_unindented_logical_line = '' self.tokens = [] self.blank_lines = self.blank_before = 0 parens = 0 @@ -1711,6 +1962,7 @@ def print_benchmark(self): class FileReport(BaseReport): """Collect the results of the checks and print only the filenames.""" + print_filename = True @@ -1917,7 +2169,8 @@ def get_checks(self, argument_name): return sorted(checks) -def get_parser(prog='pep8', version=__version__): +def get_parser(prog='pycodestyle', version=__version__): + """Create the parser for the program.""" parser = OptionParser(prog=prog, version=version, usage="%prog [options] input ...") parser.config_options = [ @@ -1979,12 +2232,12 @@ def get_parser(prog='pep8', version=__version__): def read_config(options, args, arglist, parser): - """Read and parse configurations + """Read and parse configurations. If a config file is specified on the command line with the "--config" option, then only it is used for configuration. - Otherwise, the user configuration (~/.config/pep8) and any local + Otherwise, the user configuration (~/.config/pycodestyle) and any local configurations in the current directory or above will be merged together (in that order) using the read method of ConfigParser. """ @@ -2013,8 +2266,14 @@ def read_config(options, args, arglist, parser): print('cli configuration: %s' % cli_conf) config.read(cli_conf) - pep8_section = parser.prog - if config.has_section(pep8_section): + pycodestyle_section = None + if config.has_section(parser.prog): + pycodestyle_section = parser.prog + elif config.has_section('pep8'): + pycodestyle_section = 'pep8' # Deprecated + warnings.warn('[pep8] section is deprecated. Use [pycodestyle].') + + if pycodestyle_section: option_list = dict([(o.dest, o.type or o.action) for o in parser.option_list]) @@ -2022,23 +2281,23 @@ def read_config(options, args, arglist, parser): (new_options, __) = parser.parse_args([]) # Second, parse the configuration - for opt in config.options(pep8_section): + for opt in config.options(pycodestyle_section): if opt.replace('_', '-') not in parser.config_options: print(" unknown option '%s' ignored" % opt) continue if options.verbose > 1: - print(" %s = %s" % (opt, config.get(pep8_section, opt))) + print(" %s = %s" % (opt, + config.get(pycodestyle_section, opt))) normalized_opt = opt.replace('-', '_') opt_type = option_list[normalized_opt] if opt_type in ('int', 'count'): - value = config.getint(pep8_section, opt) - elif opt_type == 'string': - value = config.get(pep8_section, opt) + value = config.getint(pycodestyle_section, opt) + elif opt_type in ('store_true', 'store_false'): + value = config.getboolean(pycodestyle_section, opt) + else: + value = config.get(pycodestyle_section, opt) if normalized_opt == 'exclude': value = normalize_paths(value, local_dir) - else: - assert opt_type in ('store_true', 'store_false') - value = config.getboolean(pep8_section, opt) setattr(new_options, normalized_opt, value) # Third, overwrite with the command-line options @@ -2052,7 +2311,7 @@ def process_options(arglist=None, parse_argv=False, config_file=None, """Process options passed either via arglist or via command line args. Passing in the ``config_file`` parameter allows other tools, such as flake8 - to specify their own options to be processed in pep8. + to specify their own options to be processed in pycodestyle. """ if not parser: parser = get_parser() @@ -2124,14 +2383,14 @@ def _main(): except AttributeError: pass # not supported on Windows - pep8style = StyleGuide(parse_argv=True) - options = pep8style.options + style_guide = StyleGuide(parse_argv=True) + options = style_guide.options if options.doctest or options.testsuite: from testsuite.support import run_tests - report = run_tests(pep8style) + report = run_tests(style_guide) else: - report = pep8style.check_files() + report = style_guide.check_files() if options.statistics: report.print_statistics() @@ -2147,5 +2406,6 @@ def _main(): sys.stderr.write(str(report.total_errors) + '\n') sys.exit(1) + if __name__ == '__main__': - _main() + _main() \ No newline at end of file diff --git a/contrib/pydocstyle.py b/contrib/pydocstyle.py deleted file mode 100755 index 7590ca8..0000000 --- a/contrib/pydocstyle.py +++ /dev/null @@ -1,1678 +0,0 @@ -#! /usr/bin/env python -"""Static analysis tool for checking docstring conventions and style. - -The repository is located at: -http://github.com/PyCQA/pydocstyle - -""" -from __future__ import with_statement - -import os -import string -import sys -import ast -import copy -import logging -import tokenize as tk -from itertools import takewhile, dropwhile, chain -from re import compile as re -import itertools -from collections import defaultdict, namedtuple, Set - -try: # Python 3.x - from ConfigParser import RawConfigParser -except ImportError: # Python 2.x - from configparser import RawConfigParser - -log = logging.getLogger(__name__) - - -try: - from StringIO import StringIO -except ImportError: # Python 3.0 and later - from io import StringIO - - -try: - next -except NameError: # Python 2.5 and earlier - nothing = object() - - def next(obj, default=nothing): - if default == nothing: - return obj.next() - else: - try: - return obj.next() - except StopIteration: - return default - - -# If possible (python >= 3.2) use tokenize.open to open files, so PEP 263 -# encoding markers are interpreted. -try: - tokenize_open = tk.open -except AttributeError: - tokenize_open = open - - -__version__ = '1.0.0' -__all__ = ('check',) - -NO_VIOLATIONS_RETURN_CODE = 0 -VIOLATIONS_RETURN_CODE = 1 -INVALID_OPTIONS_RETURN_CODE = 2 -VARIADIC_MAGIC_METHODS = ('__init__', '__call__', '__new__') - - -def humanize(string): - return re(r'(.)([A-Z]+)').sub(r'\1 \2', string).lower() - - -def is_magic(name): - return (name.startswith('__') and - name.endswith('__') and - name not in VARIADIC_MAGIC_METHODS) - - -def is_ascii(string): - return all(ord(char) < 128 for char in string) - - -def is_blank(string): - return not string.strip() - - -def leading_space(string): - return re('\s*').match(string).group() - - -class Value(object): - - def __init__(self, *args): - vars(self).update(zip(self._fields, args)) - - def __hash__(self): - return hash(repr(self)) - - def __eq__(self, other): - return other and vars(self) == vars(other) - - def __repr__(self): - kwargs = ', '.join('{0}={1!r}'.format(field, getattr(self, field)) - for field in self._fields) - return '{0}({1})'.format(self.__class__.__name__, kwargs) - - -class Definition(Value): - - _fields = ('name', '_source', 'start', 'end', 'decorators', 'docstring', - 'children', 'parent') - - _human = property(lambda self: humanize(type(self).__name__)) - kind = property(lambda self: self._human.split()[-1]) - module = property(lambda self: self.parent.module) - all = property(lambda self: self.module.all) - _slice = property(lambda self: slice(self.start - 1, self.end)) - is_class = False - - def __iter__(self): - return chain([self], *self.children) - - @property - def _publicity(self): - return {True: 'public', False: 'private'}[self.is_public] - - @property - def source(self): - """Return the source code for the definition.""" - full_src = self._source[self._slice] - - def is_empty_or_comment(line): - return line.strip() == '' or line.strip().startswith('#') - - filtered_src = dropwhile(is_empty_or_comment, reversed(full_src)) - return ''.join(reversed(list(filtered_src))) - - def __str__(self): - return 'in %s %s `%s`' % (self._publicity, self._human, self.name) - - -class Module(Definition): - - _fields = ('name', '_source', 'start', 'end', 'decorators', 'docstring', - 'children', 'parent', '_all', 'future_imports') - is_public = True - _nest = staticmethod(lambda s: {'def': Function, 'class': Class}[s]) - module = property(lambda self: self) - all = property(lambda self: self._all) - - def __init__(self, *args, **kwargs): - super(Module, self).__init__(*args, **kwargs) - self.name = self.name.lower() - - def __str__(self): - return 'at module level' - - -class Package(Module): - """A package is a __init__.py module.""" - - -class Function(Definition): - - _nest = staticmethod(lambda s: {'def': NestedFunction, - 'class': NestedClass}[s]) - - @property - def is_public(self): - if self.all is not None: - return self.name in self.all - else: - return not self.name.startswith('_') - - -class NestedFunction(Function): - - is_public = False - - -class Method(Function): - - @property - def is_public(self): - # Check if we are a setter/deleter method, and mark as private if so. - for decorator in self.decorators: - # Given 'foo', match 'foo.bar' but not 'foobar' or 'sfoo' - if re(r"^{0}\.".format(self.name)).match(decorator.name): - return False - name_is_public = (not self.name.startswith('_') or - self.name in VARIADIC_MAGIC_METHODS or - is_magic(self.name)) - return self.parent.is_public and name_is_public - - -class Class(Definition): - - _nest = staticmethod(lambda s: {'def': Method, 'class': NestedClass}[s]) - is_public = Function.is_public - is_class = True - - -class NestedClass(Class): - - @property - def is_public(self): - return (not self.name.startswith('_') and - self.parent.is_class and - self.parent.is_public) - - -class Decorator(Value): - """A decorator for function, method or class.""" - - _fields = 'name arguments'.split() - - -class TokenKind(int): - def __repr__(self): - return "tk.{0}".format(tk.tok_name[self]) - - -class Token(Value): - - _fields = 'kind value start end source'.split() - - def __init__(self, *args): - super(Token, self).__init__(*args) - self.kind = TokenKind(self.kind) - - -class TokenStream(object): - - def __init__(self, filelike): - self._generator = tk.generate_tokens(filelike.readline) - self.current = Token(*next(self._generator, None)) - self.line = self.current.start[0] - - def move(self): - previous = self.current - current = next(self._generator, None) - self.current = None if current is None else Token(*current) - self.line = self.current.start[0] if self.current else self.line - return previous - - def __iter__(self): - while True: - if self.current is not None: - yield self.current - else: - return - self.move() - - -class AllError(Exception): - - def __init__(self, message): - Exception.__init__( - self, message + - 'That means pydocstyle cannot decide which definitions are public.' - ' Variable __all__ should be present at most once in each file, ' - "in form `__all__ = ('a_public_function', 'APublicClass', ...)`. " - 'More info on __all__: http://stackoverflow.com/q/44834/. ') - - -class Parser(object): - - def __call__(self, filelike, filename): - self.source = filelike.readlines() - src = ''.join(self.source) - self.stream = TokenStream(StringIO(src)) - self.filename = filename - self.all = None - self.future_imports = defaultdict(lambda: False) - self._accumulated_decorators = [] - return self.parse_module() - - current = property(lambda self: self.stream.current) - line = property(lambda self: self.stream.line) - - def consume(self, kind): - """Consume one token and verify it is of the expected kind.""" - next_token = self.stream.move() - assert next_token.kind == kind - - def leapfrog(self, kind, value=None): - """Skip tokens in the stream until a certain token kind is reached. - - If `value` is specified, tokens whose values are different will also - be skipped. - """ - while self.current is not None: - if (self.current.kind == kind and - (value is None or self.current.value == value)): - self.consume(kind) - return - self.stream.move() - - def parse_docstring(self): - """Parse a single docstring and return its value.""" - log.debug("parsing docstring, token is %r (%s)", - self.current.kind, self.current.value) - while self.current.kind in (tk.COMMENT, tk.NEWLINE, tk.NL): - self.stream.move() - log.debug("parsing docstring, token is %r (%s)", - self.current.kind, self.current.value) - if self.current.kind == tk.STRING: - docstring = self.current.value - self.stream.move() - return docstring - return None - - def parse_decorators(self): - """Called after first @ is found. - - Parse decorators into self._accumulated_decorators. - Continue to do so until encountering the 'def' or 'class' start token. - """ - name = [] - arguments = [] - at_arguments = False - - while self.current is not None: - if (self.current.kind == tk.NAME and - self.current.value in ['def', 'class']): - # Done with decorators - found function or class proper - break - elif self.current.kind == tk.OP and self.current.value == '@': - # New decorator found. Store the decorator accumulated so far: - self._accumulated_decorators.append( - Decorator(''.join(name), ''.join(arguments))) - # Now reset to begin accumulating the new decorator: - name = [] - arguments = [] - at_arguments = False - elif self.current.kind == tk.OP and self.current.value == '(': - at_arguments = True - elif self.current.kind == tk.OP and self.current.value == ')': - # Ignore close parenthesis - pass - elif self.current.kind == tk.NEWLINE or self.current.kind == tk.NL: - # Ignore newlines - pass - else: - # Keep accumulating current decorator's name or argument. - if not at_arguments: - name.append(self.current.value) - else: - arguments.append(self.current.value) - self.stream.move() - - # Add decorator accumulated so far - self._accumulated_decorators.append( - Decorator(''.join(name), ''.join(arguments))) - - def parse_definitions(self, class_, all=False): - """Parse multiple definitions and yield them.""" - while self.current is not None: - log.debug("parsing definition list, current token is %r (%s)", - self.current.kind, self.current.value) - if all and self.current.value == '__all__': - self.parse_all() - elif self.current.kind == tk.OP and self.current.value == '@': - self.consume(tk.OP) - self.parse_decorators() - elif self.current.value in ['def', 'class']: - yield self.parse_definition(class_._nest(self.current.value)) - elif self.current.kind == tk.INDENT: - self.consume(tk.INDENT) - for definition in self.parse_definitions(class_): - yield definition - elif self.current.kind == tk.DEDENT: - self.consume(tk.DEDENT) - return - elif self.current.value == 'from': - self.parse_from_import_statement() - else: - self.stream.move() - - def parse_all(self): - """Parse the __all__ definition in a module.""" - assert self.current.value == '__all__' - self.consume(tk.NAME) - if self.current.value != '=': - raise AllError('Could not evaluate contents of __all__. ') - self.consume(tk.OP) - if self.current.value not in '([': - raise AllError('Could not evaluate contents of __all__. ') - if self.current.value == '[': - msg = ("%s WARNING: __all__ is defined as a list, this means " - "pydocstyle cannot reliably detect contents of the __all__ " - "variable, because it can be mutated. Change __all__ to be " - "an (immutable) tuple, to remove this warning. Note, " - "pydocstyle uses __all__ to detect which definitions are " - "public, to warn if public definitions are missing " - "docstrings. If __all__ is a (mutable) list, pydocstyle " - "cannot reliably assume its contents. pydocstyle will " - "proceed assuming __all__ is not mutated.\n" - % self.filename) - sys.stderr.write(msg) - self.consume(tk.OP) - - self.all = [] - all_content = "(" - while self.current.kind != tk.OP or self.current.value not in ")]": - if self.current.kind in (tk.NL, tk.COMMENT): - pass - elif (self.current.kind == tk.STRING or - self.current.value == ','): - all_content += self.current.value - else: - raise AllError('Unexpected token kind in __all__: %r. ' % - self.current.kind) - self.stream.move() - self.consume(tk.OP) - all_content += ")" - try: - self.all = eval(all_content, {}) - except BaseException as e: - raise AllError('Could not evaluate contents of __all__.' - '\bThe value was %s. The exception was:\n%s' - % (all_content, e)) - - def parse_module(self): - """Parse a module (and its children) and return a Module object.""" - log.debug("parsing module.") - start = self.line - docstring = self.parse_docstring() - children = list(self.parse_definitions(Module, all=True)) - assert self.current is None, self.current - end = self.line - cls = Module - if self.filename.endswith('__init__.py'): - cls = Package - module = cls(self.filename, self.source, start, end, - [], docstring, children, None, self.all) - for child in module.children: - child.parent = module - module.future_imports = self.future_imports - log.debug("finished parsing module.") - return module - - def parse_definition(self, class_): - """Parse a definition and return its value in a `class_` object.""" - start = self.line - self.consume(tk.NAME) - name = self.current.value - log.debug("parsing %s '%s'", class_.__name__, name) - self.stream.move() - if self.current.kind == tk.OP and self.current.value == '(': - parenthesis_level = 0 - while True: - if self.current.kind == tk.OP: - if self.current.value == '(': - parenthesis_level += 1 - elif self.current.value == ')': - parenthesis_level -= 1 - if parenthesis_level == 0: - break - self.stream.move() - if self.current.kind != tk.OP or self.current.value != ':': - self.leapfrog(tk.OP, value=":") - else: - self.consume(tk.OP) - if self.current.kind in (tk.NEWLINE, tk.COMMENT): - self.leapfrog(tk.INDENT) - assert self.current.kind != tk.INDENT - docstring = self.parse_docstring() - decorators = self._accumulated_decorators - self._accumulated_decorators = [] - log.debug("parsing nested definitions.") - children = list(self.parse_definitions(class_)) - log.debug("finished parsing nested definitions for '%s'", name) - end = self.line - 1 - else: # one-liner definition - docstring = self.parse_docstring() - decorators = [] # TODO - children = [] - end = self.line - self.leapfrog(tk.NEWLINE) - definition = class_(name, self.source, start, end, - decorators, docstring, children, None) - for child in definition.children: - child.parent = definition - log.debug("finished parsing %s '%s'. Next token is %r (%s)", - class_.__name__, name, self.current.kind, - self.current.value) - return definition - - def parse_from_import_statement(self): - """Parse a 'from x import y' statement. - - The purpose is to find __future__ statements. - - """ - log.debug('parsing from/import statement.') - assert self.current.value == 'from', self.current.value - self.stream.move() - if self.current.value != '__future__': - return - self.stream.move() - assert self.current.value == 'import', self.current.value - self.stream.move() - if self.current.value == '(': - self.consume(tk.OP) - expected_end_kind = tk.OP - else: - expected_end_kind = tk.NEWLINE - while self.current.kind != expected_end_kind and not( - self.current.kind == tk.OP and self.current.value == ';'): - if self.current.kind != tk.NAME: - self.stream.move() - continue - log.debug("parsing import, token is %r (%s)", - self.current.kind, self.current.value) - log.debug('found future import: %s', self.current.value) - self.future_imports[self.current.value] = True - self.consume(tk.NAME) - log.debug("parsing import, token is %r (%s)", - self.current.kind, self.current.value) - if self.current.kind == tk.NAME and self.current.value == 'as': - self.consume(tk.NAME) # as - if self.current.kind == tk.NAME: - self.consume(tk.NAME) # new name, irrelevant - if self.current.value == ',': - self.consume(tk.OP) - log.debug("parsing import, token is %r (%s)", - self.current.kind, self.current.value) - - -class Error(object): - """Error in docstring style.""" - - # should be overridden by inheriting classes - code = None - short_desc = None - context = None - - # Options that define how errors are printed: - explain = False - source = False - - def __init__(self, *parameters): - self.parameters = parameters - self.definition = None - self.explanation = None - - def set_context(self, definition, explanation): - self.definition = definition - self.explanation = explanation - - filename = property(lambda self: self.definition.module.name) - line = property(lambda self: self.definition.start) - - @property - def message(self): - ret = '%s: %s' % (self.code, self.short_desc) - if self.context is not None: - ret += ' (' + self.context % self.parameters + ')' - return ret - - @property - def lines(self): - source = '' - lines = self.definition._source[self.definition._slice] - offset = self.definition.start - lines_stripped = list(reversed(list(dropwhile(is_blank, - reversed(lines))))) - numbers_width = 0 - for n, line in enumerate(lines_stripped): - numbers_width = max(numbers_width, n + offset) - numbers_width = len(str(numbers_width)) - numbers_width = 6 - for n, line in enumerate(lines_stripped): - source += '%*d: %s' % (numbers_width, n + offset, line) - if n > 5: - source += ' ...\n' - break - return source - - def __str__(self): - self.explanation = '\n'.join(l for l in self.explanation.split('\n') - if not is_blank(l)) - template = '%(filename)s:%(line)s %(definition)s:\n %(message)s' - if self.source and self.explain: - template += '\n\n%(explanation)s\n\n%(lines)s\n' - elif self.source and not self.explain: - template += '\n\n%(lines)s\n' - elif self.explain and not self.source: - template += '\n\n%(explanation)s\n\n' - return template % dict((name, getattr(self, name)) for name in - ['filename', 'line', 'definition', 'message', - 'explanation', 'lines']) - - __repr__ = __str__ - - def __lt__(self, other): - return (self.filename, self.line) < (other.filename, other.line) - - -class ErrorRegistry(object): - groups = [] - - class ErrorGroup(object): - - def __init__(self, prefix, name): - self.prefix = prefix - self.name = name - self.errors = [] - - def create_error(self, error_code, error_desc, error_context=None): - # TODO: check prefix - - class _Error(Error): - code = error_code - short_desc = error_desc - context = error_context - - self.errors.append(_Error) - return _Error - - @classmethod - def create_group(cls, prefix, name): - group = cls.ErrorGroup(prefix, name) - cls.groups.append(group) - return group - - @classmethod - def get_error_codes(cls): - for group in cls.groups: - for error in group.errors: - yield error.code - - @classmethod - def to_rst(cls): - sep_line = '+' + 6 * '-' + '+' + '-' * 71 + '+\n' - blank_line = '|' + 78 * ' ' + '|\n' - table = '' - for group in cls.groups: - table += sep_line - table += blank_line - table += '|' + ('**%s**' % group.name).center(78) + '|\n' - table += blank_line - for error in group.errors: - table += sep_line - table += ('|' + error.code.center(6) + '| ' + - error.short_desc.ljust(70) + '|\n') - table += sep_line - return table - - -D1xx = ErrorRegistry.create_group('D1', 'Missing Docstrings') -D100 = D1xx.create_error('D100', 'Missing docstring in public module') -D101 = D1xx.create_error('D101', 'Missing docstring in public class') -D102 = D1xx.create_error('D102', 'Missing docstring in public method') -D103 = D1xx.create_error('D103', 'Missing docstring in public function') -D104 = D1xx.create_error('D104', 'Missing docstring in public package') -D105 = D1xx.create_error('D105', 'Missing docstring in magic method') - -D2xx = ErrorRegistry.create_group('D2', 'Whitespace Issues') -D200 = D2xx.create_error('D200', 'One-line docstring should fit on one line ' - 'with quotes', 'found %s') -D201 = D2xx.create_error('D201', 'No blank lines allowed before function ' - 'docstring', 'found %s') -D202 = D2xx.create_error('D202', 'No blank lines allowed after function ' - 'docstring', 'found %s') -D203 = D2xx.create_error('D203', '1 blank line required before class ' - 'docstring', 'found %s') -D204 = D2xx.create_error('D204', '1 blank line required after class ' - 'docstring', 'found %s') -D205 = D2xx.create_error('D205', '1 blank line required between summary line ' - 'and description', 'found %s') -D206 = D2xx.create_error('D206', 'Docstring should be indented with spaces, ' - 'not tabs') -D207 = D2xx.create_error('D207', 'Docstring is under-indented') -D208 = D2xx.create_error('D208', 'Docstring is over-indented') -D209 = D2xx.create_error('D209', 'Multi-line docstring closing quotes should ' - 'be on a separate line') -D210 = D2xx.create_error('D210', 'No whitespaces allowed surrounding ' - 'docstring text') -D211 = D2xx.create_error('D211', 'No blank lines allowed before class ' - 'docstring', 'found %s') - -D3xx = ErrorRegistry.create_group('D3', 'Quotes Issues') -D300 = D3xx.create_error('D300', 'Use """triple double quotes"""', - 'found %s-quotes') -D301 = D3xx.create_error('D301', 'Use r""" if any backslashes in a docstring') -D302 = D3xx.create_error('D302', 'Use u""" for Unicode docstrings') - -D4xx = ErrorRegistry.create_group('D4', 'Docstring Content Issues') -D400 = D4xx.create_error('D400', 'First line should end with a period', - 'not %r') -D401 = D4xx.create_error('D401', 'First line should be in imperative mood', - '%r, not %r') -D402 = D4xx.create_error('D402', 'First line should not be the function\'s ' - '"signature"') -D403 = D4xx.create_error('D403', 'First word of the first line should be ' - 'properly capitalized', '%r, not %r') - - -class AttrDict(dict): - def __getattr__(self, item): - return self[item] - - -conventions = AttrDict({ - 'pep257': set(ErrorRegistry.get_error_codes()) - set(['D203']), -}) - - -# General configurations for pydocstyle run. -RunConfiguration = namedtuple('RunConfiguration', - ('explain', 'source', 'debug', - 'verbose', 'count')) - - -class IllegalConfiguration(Exception): - """An exception for illegal configurations.""" - - pass - - -# Check configuration - used by the ConfigurationParser class. -CheckConfiguration = namedtuple('CheckConfiguration', - ('checked_codes', 'match', 'match_dir')) - - -def check_initialized(method): - """Check that the configuration object was initialized.""" - def _decorator(self, *args, **kwargs): - if self._arguments is None or self._options is None: - raise RuntimeError('using an uninitialized configuration') - return method(self, *args, **kwargs) - return _decorator - - -class ConfigurationParser(object): - """Responsible for parsing configuration from files and CLI. - - There are 2 types of configurations: Run configurations and Check - configurations. - - Run Configurations: - ------------------ - Responsible for deciding things that are related to the user interface, - e.g. verbosity, debug options, etc. - All run configurations default to `False` and are decided only by CLI. - - Check Configurations: - -------------------- - Configurations that are related to which files and errors will be checked. - These are configurable in 2 ways: using the CLI, and using configuration - files. - - Configuration files are nested within the file system, meaning that the - closer a configuration file is to a checked file, the more relevant it will - be. For instance, imagine this directory structure: - - A - +-- tox.ini: sets `select=D100` - +-- B - +-- foo.py - +-- tox.ini: sets `add-ignore=D100` - - Then `foo.py` will not be checked for `D100`. - The configuration build algorithm is described in `self._get_config`. - - Note: If any of `BASE_ERROR_SELECTION_OPTIONS` was selected in the CLI, all - configuration files will be ignored and each file will be checked for - the error codes supplied in the CLI. - - """ - - CONFIG_FILE_OPTIONS = ('convention', 'select', 'ignore', 'add-select', - 'add-ignore', 'match', 'match-dir') - BASE_ERROR_SELECTION_OPTIONS = ('ignore', 'select', 'convention') - - DEFAULT_MATCH_RE = '(?!test_).*\.py' - DEFAULT_MATCH_DIR_RE = '[^\.].*' - DEFAULT_CONVENTION = conventions.pep257 - - PROJECT_CONFIG_FILES = ( - 'setup.cfg', - 'tox.ini', - '.pydocstyle', - '.pydocstylerc', - # The following is deprecated, but remains for backwards compatibility. - '.pep257', - ) - - POSSIBLE_SECTION_NAMES = ('pydocstyle', 'pep257') - - def __init__(self): - """Create a configuration parser.""" - self._cache = {} - self._override_by_cli = None - self._options = self._arguments = self._run_conf = None - self._parser = self._create_option_parser() - - # ---------------------------- Public Methods ----------------------------- - - def get_default_run_configuration(self): - """Return a `RunConfiguration` object set with default values.""" - options, _ = self._parse_args([]) - return self._create_run_config(options) - - def parse(self): - """Parse the configuration. - - If one of `BASE_ERROR_SELECTION_OPTIONS` was selected, overrides all - error codes to check and disregards any error code related - configurations from the configuration files. - - """ - self._options, self._arguments = self._parse_args() - self._arguments = self._arguments or ['.'] - - if not self._validate_options(self._options): - raise IllegalConfiguration() - - self._run_conf = self._create_run_config(self._options) - - config = self._create_check_config(self._options, use_dafaults=False) - self._override_by_cli = config - - @check_initialized - def get_user_run_configuration(self): - """Return the run configuration for the script.""" - return self._run_conf - - @check_initialized - def get_files_to_check(self): - """Generate files and error codes to check on each one. - - Walk dir trees under `self._arguments` and generate yield filnames - that `match` under each directory that `match_dir`. - The method locates the configuration for each file name and yields a - tuple of (filename, [error_codes]). - - With every discovery of a new configuration file `IllegalConfiguration` - might be raised. - - """ - def _get_matches(config): - """Return the `match` and `match_dir` functions for `config`.""" - match_func = re(config.match + '$').match - match_dir_func = re(config.match_dir + '$').match - return match_func, match_dir_func - - for name in self._arguments: - if os.path.isdir(name): - for root, dirs, filenames in os.walk(name): - config = self._get_config(root) - match, match_dir = _get_matches(config) - - # Skip any dirs that do not match match_dir - dirs[:] = [dir for dir in dirs if match_dir(dir)] - - for filename in filenames: - if match(filename): - full_path = os.path.join(root, filename) - yield full_path, list(config.checked_codes) - else: - config = self._get_config(name) - match, _ = _get_matches(config) - if match(name): - yield name, list(config.checked_codes) - - # --------------------------- Private Methods ----------------------------- - - def _get_config(self, node): - """Get and cache the run configuration for `node`. - - If no configuration exists (not local and not for the parend node), - returns and caches a default configuration. - - The algorithm: - ------------- - * If the current directory's configuration exists in - `self._cache` - return it. - * If a configuration file does not exist in this directory: - * If the directory is not a root directory: - * Cache its configuration as this directory's and return it. - * Else: - * Cache a default configuration and return it. - * Else: - * Read the configuration file. - * If a parent directory exists AND the configuration file - allows inheritance: - * Read the parent configuration by calling this function with the - parent directory as `node`. - * Merge the parent configuration with the current one and - cache it. - * If the user has specified one of `BASE_ERROR_SELECTION_OPTIONS` in - the CLI - return the CLI configuration with the configuration match - clauses - * Set the `--add-select` and `--add-ignore` CLI configurations. - - """ - path = os.path.abspath(node) - path = path if os.path.isdir(path) else os.path.dirname(path) - - if path in self._cache: - return self._cache[path] - - config_file = self._get_config_file_in_folder(path) - - if config_file is None: - parent_dir, tail = os.path.split(path) - if tail: - # No configuration file, simply take the parent's. - config = self._get_config(parent_dir) - else: - # There's no configuration file and no parent directory. - # Use the default configuration or the one given in the CLI. - config = self._create_check_config(self._options) - else: - # There's a config file! Read it and merge if necessary. - options, inherit = self._read_configuration_file(config_file) - - parent_dir, tail = os.path.split(path) - if tail and inherit: - # There is a parent dir and we should try to merge. - parent_config = self._get_config(parent_dir) - config = self._merge_configuration(parent_config, options) - else: - # No need to merge or parent dir does not exist. - config = self._create_check_config(options) - - # Make the CLI always win - final_config = {} - for attr in CheckConfiguration._fields: - cli_val = getattr(self._override_by_cli, attr) - conf_val = getattr(config, attr) - final_config[attr] = cli_val if cli_val is not None else conf_val - - config = CheckConfiguration(**final_config) - - self._set_add_options(config.checked_codes, self._options) - self._cache[path] = config - return self._cache[path] - - def _read_configuration_file(self, path): - """Try to read and parse `path` as a configuration file. - - If the configurations were illegal (checked with - `self._validate_options`), raises `IllegalConfiguration`. - - Returns (options, should_inherit). - - """ - parser = RawConfigParser() - options = None - should_inherit = True - - if parser.read(path) and self._get_section_name(parser): - option_list = dict([(o.dest, o.type or o.action) - for o in self._parser.option_list]) - - # First, read the default values - new_options, _ = self._parse_args([]) - - # Second, parse the configuration - section_name = self._get_section_name(parser) - for opt in parser.options(section_name): - if opt == 'inherit': - should_inherit = parser.getboolean(section_name, opt) - continue - - if opt.replace('_', '-') not in self.CONFIG_FILE_OPTIONS: - log.warning("Unknown option '{0}' ignored".format(opt)) - continue - - normalized_opt = opt.replace('-', '_') - opt_type = option_list[normalized_opt] - if opt_type in ('int', 'count'): - value = parser.getint(section_name, opt) - elif opt_type == 'string': - value = parser.get(section_name, opt) - else: - assert opt_type in ('store_true', 'store_false') - value = parser.getboolean(section_name, opt) - setattr(new_options, normalized_opt, value) - - # Third, fix the set-options - options = self._fix_set_options(new_options) - - if options is not None: - if not self._validate_options(options): - raise IllegalConfiguration('in file: {0}'.format(path)) - - return options, should_inherit - - def _merge_configuration(self, parent_config, child_options): - """Merge parent config into the child options. - - The migration process requires an `options` object for the child in - order to distinguish between mutually exclusive codes, add-select and - add-ignore error codes. - - """ - # Copy the parent error codes so we won't override them - error_codes = copy.deepcopy(parent_config.checked_codes) - if self._has_exclusive_option(child_options): - error_codes = self._get_exclusive_error_codes(child_options) - - self._set_add_options(error_codes, child_options) - - match = child_options.match \ - if child_options.match is not None else parent_config.match - match_dir = child_options.match_dir \ - if child_options.match_dir is not None else parent_config.match_dir - - return CheckConfiguration(checked_codes=error_codes, - match=match, - match_dir=match_dir) - - def _parse_args(self, args=None, values=None): - """Parse the options using `self._parser` and reformat the options.""" - options, arguments = self._parser.parse_args(args, values) - return self._fix_set_options(options), arguments - - @staticmethod - def _create_run_config(options): - """Create a `RunConfiguration` object from `options`.""" - values = dict([(opt, getattr(options, opt)) for opt in - RunConfiguration._fields]) - return RunConfiguration(**values) - - @classmethod - def _create_check_config(cls, options, use_dafaults=True): - """Create a `CheckConfiguration` object from `options`. - - If `use_dafaults`, any of the match options that are `None` will - be replaced with their default value and the default convention will be - set for the checked codes. - - """ - match = cls.DEFAULT_MATCH_RE \ - if options.match is None and use_dafaults \ - else options.match - - match_dir = cls.DEFAULT_MATCH_DIR_RE \ - if options.match_dir is None and use_dafaults \ - else options.match_dir - - checked_codes = None - - if cls._has_exclusive_option(options) or use_dafaults: - checked_codes = cls._get_checked_errors(options) - - return CheckConfiguration(checked_codes=checked_codes, - match=match, match_dir=match_dir) - - @classmethod - def _get_section_name(cls, parser): - """Parse options from relevant section.""" - for section_name in cls.POSSIBLE_SECTION_NAMES: - if parser.has_section(section_name): - return section_name - - return None - - @classmethod - def _get_config_file_in_folder(cls, path): - """Look for a configuration file in `path`. - - If exists return it's full path, otherwise None. - - """ - if os.path.isfile(path): - path = os.path.dirname(path) - - for fn in cls.PROJECT_CONFIG_FILES: - config = RawConfigParser() - full_path = os.path.join(path, fn) - if config.read(full_path) and cls._get_section_name(config): - return full_path - - @staticmethod - def _get_exclusive_error_codes(options): - """Extract the error codes from the selected exclusive option.""" - codes = set(ErrorRegistry.get_error_codes()) - checked_codes = None - - if options.ignore is not None: - checked_codes = codes - options.ignore - elif options.select is not None: - checked_codes = options.select - elif options.convention is not None: - checked_codes = getattr(conventions, options.convention) - - # To not override the conventions nor the options - copy them. - return copy.deepcopy(checked_codes) - - @staticmethod - def _set_add_options(checked_codes, options): - """Set `checked_codes` by the `add_ignore` or `add_select` options.""" - checked_codes |= options.add_select - checked_codes -= options.add_ignore - - @classmethod - def _get_checked_errors(cls, options): - """Extract the codes needed to be checked from `options`.""" - checked_codes = cls._get_exclusive_error_codes(options) - if checked_codes is None: - checked_codes = cls.DEFAULT_CONVENTION - - cls._set_add_options(checked_codes, options) - - return checked_codes - - @classmethod - def _validate_options(cls, options): - """Validate the mutually exclusive options. - - Return `True` iff only zero or one of `BASE_ERROR_SELECTION_OPTIONS` - was selected. - - """ - for opt1, opt2 in \ - itertools.permutations(cls.BASE_ERROR_SELECTION_OPTIONS, 2): - if getattr(options, opt1) and getattr(options, opt2): - log.error('Cannot pass both {0} and {1}. They are ' - 'mutually exclusive.'.format(opt1, opt2)) - return False - - if options.convention and options.convention not in conventions: - log.error("Illegal convention '{0}'. Possible conventions: {1}" - .format(options.convention, - ', '.join(conventions.keys()))) - return False - return True - - @classmethod - def _has_exclusive_option(cls, options): - """Return `True` iff one or more exclusive options were selected.""" - return any([getattr(options, opt) is not None for opt in - cls.BASE_ERROR_SELECTION_OPTIONS]) - - @staticmethod - def _fix_set_options(options): - """Alter the set options from None/strings to sets in place.""" - optional_set_options = ('ignore', 'select') - mandatory_set_options = ('add_ignore', 'add_select') - - def _get_set(value_str): - """Split `value_str` by the delimiter `,` and return a set. - - Removes any occurrences of '' in the set. - - """ - return set(value_str.split(',')) - set(['']) - - for opt in optional_set_options: - value = getattr(options, opt) - if value is not None: - setattr(options, opt, _get_set(value)) - - for opt in mandatory_set_options: - value = getattr(options, opt) - if value is None: - value = '' - - if not isinstance(value, Set): - value = _get_set(value) - - setattr(options, opt, value) - - return options - - @classmethod - def _create_option_parser(cls): - """Return an option parser to parse the command line arguments.""" - from optparse import OptionParser - - parser = OptionParser( - version=__version__, - usage='Usage: pydocstyle [options] [...]') - - option = parser.add_option - - # Run configuration options - option('-e', '--explain', action='store_true', default=False, - help='show explanation of each error') - option('-s', '--source', action='store_true', default=False, - help='show source for each error') - option('-d', '--debug', action='store_true', default=False, - help='print debug information') - option('-v', '--verbose', action='store_true', default=False, - help='print status information') - option('--count', action='store_true', default=False, - help='print total number of errors to stdout') - - # Error check options - option('--select', metavar='', default=None, - help='choose the basic list of checked errors by ' - 'specifying which errors to check for (with a list of ' - 'comma-separated error codes). ' - 'for example: --select=D101,D202') - option('--ignore', metavar='', default=None, - help='choose the basic list of checked errors by ' - 'specifying which errors to ignore (with a list of ' - 'comma-separated error codes). ' - 'for example: --ignore=D101,D202') - option('--convention', metavar='', default=None, - help='choose the basic list of checked errors by specifying an ' - 'existing convention. Possible conventions: {0}' - .format(', '.join(conventions))) - option('--add-select', metavar='', default=None, - help='amend the list of errors to check for by specifying ' - 'more error codes to check.') - option('--add-ignore', metavar='', default=None, - help='amend the list of errors to check for by specifying ' - 'more error codes to ignore.') - - # Match clauses - option('--match', metavar='', default=None, - help=("check only files that exactly match regular " - "expression; default is --match='{0}' which matches " - "files that don't start with 'test_' but end with " - "'.py'").format(cls.DEFAULT_MATCH_RE)) - option('--match-dir', metavar='', default=None, - help=("search only dirs that exactly match regular " - "expression; default is --match-dir='{0}', which " - "matches all dirs that don't start with " - "a dot").format(cls.DEFAULT_MATCH_DIR_RE)) - - return parser - - -def check(filenames, select=None, ignore=None): - """Generate PEP 257 errors that exist in `filenames` iterable. - - Only returns errors with error-codes defined in `checked_codes` iterable. - - Example - ------- - >>> check([ppydocstyle.py.py], checked_codes=['D100']) - - - """ - if select is not None and ignore is not None: - raise IllegalConfiguration('Cannot pass both select and ignore. ' - 'They are mutually exclusive.') - elif select is not None: - checked_codes = select - elif ignore is not None: - checked_codes = list(set(ErrorRegistry.get_error_codes()) - - set(ignore)) - else: - checked_codes = conventions.pep257 - - for filename in filenames: - log.info('Checking file %s.', filename) - try: - with tokenize_open(filename) as file: - source = file.read() - for error in PEP257Checker().check_source(source, filename): - code = getattr(error, 'code', None) - if code in checked_codes: - yield error - except (EnvironmentError, AllError): - yield sys.exc_info()[1] - except tk.TokenError: - yield SyntaxError('invalid syntax in file %s' % filename) - - -def setup_stream_handlers(conf): - """Setup logging stream handlers according to the options.""" - class StdoutFilter(logging.Filter): - def filter(self, record): - return record.levelno in (logging.DEBUG, logging.INFO) - - log.handlers = [] - - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.setLevel(logging.WARNING) - stdout_handler.addFilter(StdoutFilter()) - if conf.debug: - stdout_handler.setLevel(logging.DEBUG) - elif conf.verbose: - stdout_handler.setLevel(logging.INFO) - else: - stdout_handler.setLevel(logging.WARNING) - log.addHandler(stdout_handler) - - stderr_handler = logging.StreamHandler(sys.stderr) - stderr_handler.setLevel(logging.WARNING) - log.addHandler(stderr_handler) - - -def run_pydocstyle(used_pep257=False): - log.setLevel(logging.DEBUG) - conf = ConfigurationParser() - setup_stream_handlers(conf.get_default_run_configuration()) - - try: - conf.parse() - except IllegalConfiguration: - return INVALID_OPTIONS_RETURN_CODE - - run_conf = conf.get_user_run_configuration() - - # Reset the logger according to the command line arguments - setup_stream_handlers(run_conf) - - if used_pep257: - log.warning("Deprecation Warning:\n" - "pep257 has been renamed to pydocstyle and the use of the " - "pep257 executable is deprecated and will be removed in " - "the next major version. Please use `pydocstyle` instead.") - - log.debug("starting in debug mode.") - - Error.explain = run_conf.explain - Error.source = run_conf.source - - errors = [] - try: - for filename, checked_codes in conf.get_files_to_check(): - errors.extend(check((filename,), select=checked_codes)) - except IllegalConfiguration: - # An illegal configuration file was found during file generation. - return INVALID_OPTIONS_RETURN_CODE - - code = NO_VIOLATIONS_RETURN_CODE - count = 0 - for error in errors: - sys.stderr.write('%s\n' % error) - code = VIOLATIONS_RETURN_CODE - count += 1 - if run_conf.count: - print(count) - return code - - -parse = Parser() - - -def check_for(kind, terminal=False): - def decorator(f): - f._check_for = kind - f._terminal = terminal - return f - return decorator - - -class PEP257Checker(object): - """Checker for PEP 257. - - D10x: Missing docstrings - D20x: Whitespace issues - D30x: Docstring formatting - D40x: Docstring content issues - - """ - - def check_source(self, source, filename): - module = parse(StringIO(source), filename) - for definition in module: - for check in self.checks: - terminate = False - if isinstance(definition, check._check_for): - error = check(None, definition, definition.docstring) - errors = error if hasattr(error, '__iter__') else [error] - for error in errors: - if error is not None: - partition = check.__doc__.partition('.\n') - message, _, explanation = partition - error.set_context(explanation=explanation, - definition=definition) - yield error - if check._terminal: - terminate = True - break - if terminate: - break - - @property - def checks(self): - all = [check for check in vars(type(self)).values() - if hasattr(check, '_check_for')] - return sorted(all, key=lambda check: not check._terminal) - - @check_for(Definition, terminal=True) - def check_docstring_missing(self, definition, docstring): - """D10{0,1,2,3}: Public definitions should have docstrings. - - All modules should normally have docstrings. [...] all functions and - classes exported by a module should also have docstrings. Public - methods (including the __init__ constructor) should also have - docstrings. - - Note: Public (exported) definitions are either those with names listed - in __all__ variable (if present), or those that do not start - with a single underscore. - - """ - if (not docstring and definition.is_public or - docstring and is_blank(ast.literal_eval(docstring))): - codes = {Module: D100, Class: D101, NestedClass: D101, - Method: (lambda: D105() if is_magic(definition.name) - else D102()), - Function: D103, NestedFunction: D103, Package: D104} - return codes[type(definition)]() - - @check_for(Definition) - def check_one_liners(self, definition, docstring): - """D200: One-liner docstrings should fit on one line with quotes. - - The closing quotes are on the same line as the opening quotes. - This looks better for one-liners. - - """ - if docstring: - lines = ast.literal_eval(docstring).split('\n') - if len(lines) > 1: - non_empty_lines = sum(1 for l in lines if not is_blank(l)) - if non_empty_lines == 1: - return D200(len(lines)) - - @check_for(Function) - def check_no_blank_before(self, function, docstring): # def - """D20{1,2}: No blank lines allowed around function/method docstring. - - There's no blank line either before or after the docstring. - - """ - # NOTE: This does not take into account functions with groups of code. - if docstring: - before, _, after = function.source.partition(docstring) - blanks_before = list(map(is_blank, before.split('\n')[:-1])) - blanks_after = list(map(is_blank, after.split('\n')[1:])) - blanks_before_count = sum(takewhile(bool, reversed(blanks_before))) - blanks_after_count = sum(takewhile(bool, blanks_after)) - if blanks_before_count != 0: - yield D201(blanks_before_count) - if not all(blanks_after) and blanks_after_count != 0: - yield D202(blanks_after_count) - - @check_for(Class) - def check_blank_before_after_class(self, class_, docstring): - """D20{3,4}: Class docstring should have 1 blank line around them. - - Insert a blank line before and after all docstrings (one-line or - multi-line) that document a class -- generally speaking, the class's - methods are separated from each other by a single blank line, and the - docstring needs to be offset from the first method by a blank line; - for symmetry, put a blank line between the class header and the - docstring. - - """ - # NOTE: this gives false-positive in this case - # class Foo: - # - # """Docstring.""" - # - # - # # comment here - # def foo(): pass - if docstring: - before, _, after = class_.source.partition(docstring) - blanks_before = list(map(is_blank, before.split('\n')[:-1])) - blanks_after = list(map(is_blank, after.split('\n')[1:])) - blanks_before_count = sum(takewhile(bool, reversed(blanks_before))) - blanks_after_count = sum(takewhile(bool, blanks_after)) - if blanks_before_count != 0: - yield D211(blanks_before_count) - if blanks_before_count != 1: - yield D203(blanks_before_count) - if not all(blanks_after) and blanks_after_count != 1: - yield D204(blanks_after_count) - - @check_for(Definition) - def check_blank_after_summary(self, definition, docstring): - """D205: Put one blank line between summary line and description. - - Multi-line docstrings consist of a summary line just like a one-line - docstring, followed by a blank line, followed by a more elaborate - description. The summary line may be used by automatic indexing tools; - it is important that it fits on one line and is separated from the - rest of the docstring by a blank line. - - """ - if docstring: - lines = ast.literal_eval(docstring).strip().split('\n') - if len(lines) > 1: - post_summary_blanks = list(map(is_blank, lines[1:])) - blanks_count = sum(takewhile(bool, post_summary_blanks)) - if blanks_count != 1: - return D205(blanks_count) - - @check_for(Definition) - def check_indent(self, definition, docstring): - """D20{6,7,8}: The entire docstring should be indented same as code. - - The entire docstring is indented the same as the quotes at its - first line. - - """ - if docstring: - before_docstring, _, _ = definition.source.partition(docstring) - _, _, indent = before_docstring.rpartition('\n') - lines = docstring.split('\n') - if len(lines) > 1: - lines = lines[1:] # First line does not need indent. - indents = [leading_space(l) for l in lines if not is_blank(l)] - if set(' \t') == set(''.join(indents) + indent): - yield D206() - if (len(indents) > 1 and min(indents[:-1]) > indent or - indents[-1] > indent): - yield D208() - if min(indents) < indent: - yield D207() - - @check_for(Definition) - def check_newline_after_last_paragraph(self, definition, docstring): - """D209: Put multi-line docstring closing quotes on separate line. - - Unless the entire docstring fits on a line, place the closing - quotes on a line by themselves. - - """ - if docstring: - lines = [l for l in ast.literal_eval(docstring).split('\n') - if not is_blank(l)] - if len(lines) > 1: - if docstring.split("\n")[-1].strip() not in ['"""', "'''"]: - return D209() - - @check_for(Definition) - def check_surrounding_whitespaces(self, definition, docstring): - """D210: No whitespaces allowed surrounding docstring text.""" - if docstring: - lines = ast.literal_eval(docstring).split('\n') - if lines[0].startswith(' ') or \ - len(lines) == 1 and lines[0].endswith(' '): - return D210() - - @check_for(Definition) - def check_triple_double_quotes(self, definition, docstring): - r'''D300: Use """triple double quotes""". - - For consistency, always use """triple double quotes""" around - docstrings. Use r"""raw triple double quotes""" if you use any - backslashes in your docstrings. For Unicode docstrings, use - u"""Unicode triple-quoted strings""". - - Note: Exception to this is made if the docstring contains - """ quotes in its body. - - ''' - if (docstring and '"""' in ast.literal_eval(docstring) and - docstring.startswith(("'''", "r'''", "u'''", "ur'''"))): - # Allow ''' quotes if docstring contains """, because otherwise """ - # quotes could not be expressed inside docstring. Not in PEP 257. - return - if docstring and not docstring.startswith( - ('"""', 'r"""', 'u"""', 'ur"""')): - quotes = "'''" if "'''" in docstring[:4] else "'" - return D300(quotes) - - @check_for(Definition) - def check_backslashes(self, definition, docstring): - r'''D301: Use r""" if any backslashes in a docstring. - - Use r"""raw triple double quotes""" if you use any backslashes - (\\) in your docstrings. - - ''' - # Just check that docstring is raw, check_triple_double_quotes - # ensures the correct quotes. - if docstring and '\\' in docstring and not docstring.startswith( - ('r', 'ur')): - return D301() - - @check_for(Definition) - def check_unicode_docstring(self, definition, docstring): - r'''D302: Use u""" for docstrings with Unicode. - - For Unicode docstrings, use u"""Unicode triple-quoted strings""". - - ''' - if definition.module.future_imports['unicode_literals']: - return - - # Just check that docstring is unicode, check_triple_double_quotes - # ensures the correct quotes. - if docstring and sys.version_info[0] <= 2: - if not is_ascii(docstring) and not docstring.startswith( - ('u', 'ur')): - return D302() - - @check_for(Definition) - def check_ends_with_period(self, definition, docstring): - """D400: First line should end with a period. - - The [first line of a] docstring is a phrase ending in a period. - - """ - if docstring: - summary_line = ast.literal_eval(docstring).strip().split('\n')[0] - if not summary_line.endswith('.'): - return D400(summary_line[-1]) - - @check_for(Function) - def check_imperative_mood(self, function, docstring): # def context - """D401: First line should be in imperative mood: 'Do', not 'Does'. - - [Docstring] prescribes the function or method's effect as a command: - ("Do this", "Return that"), not as a description; e.g. don't write - "Returns the pathname ...". - - """ - if docstring: - stripped = ast.literal_eval(docstring).strip() - if stripped: - first_word = stripped.split()[0] - if first_word.endswith('s') and not first_word.endswith('ss'): - return D401(first_word[:-1], first_word) - - @check_for(Function) - def check_no_signature(self, function, docstring): # def context - """D402: First line should not be function's or method's "signature". - - The one-line docstring should NOT be a "signature" reiterating the - function/method parameters (which can be obtained by introspection). - - """ - if docstring: - first_line = ast.literal_eval(docstring).strip().split('\n')[0] - if function.name + '(' in first_line.replace(' ', ''): - return D402() - - @check_for(Function) - def check_capitalized(self, function, docstring): - """D403: First word of the first line should be properly capitalized. - - The [first line of a] docstring is a phrase ending in a period. - - """ - if docstring: - first_word = ast.literal_eval(docstring).split()[0] - if first_word == first_word.upper(): - return - for char in first_word: - if char not in string.ascii_letters and char != "'": - return - if first_word != first_word.capitalize(): - return D403(first_word.capitalize(), first_word) - - # Somewhat hard to determine if return value is mentioned. - # @check(Function) - def SKIP_check_return_type(self, function, docstring): - """D40x: Return value type should be mentioned. - - [T]he nature of the return value cannot be determined by - introspection, so it should be mentioned. - - """ - if docstring and function.returns_value: - if 'return' not in docstring.lower(): - return Error() - - -def main(use_pep257=False): - try: - sys.exit(run_pydocstyle(use_pep257)) - except KeyboardInterrupt: - pass - - -def main_pep257(): - main(use_pep257=True) - - -if __name__ == '__main__': - main() diff --git a/contrib/pydocstyle/__init__.py b/contrib/pydocstyle/__init__.py new file mode 100644 index 0000000..04e7e76 --- /dev/null +++ b/contrib/pydocstyle/__init__.py @@ -0,0 +1,7 @@ +from .checker import check +from .violations import Error, conventions +from .utils import __version__ + +# Temporary hotfix for flake8-docstrings +from .checker import ConventionChecker, tokenize_open +from .parser import AllError diff --git a/contrib/pydocstyle/__main__.py b/contrib/pydocstyle/__main__.py new file mode 100644 index 0000000..05f0902 --- /dev/null +++ b/contrib/pydocstyle/__main__.py @@ -0,0 +1,19 @@ +#! /usr/bin/env python +"""Static analysis tool for checking docstring conventions and style. + +The repository is located at: +http://github.com/PyCQA/pydocstyle + +""" + + +__all__ = () + + +def main(): + from pydocstyle import cli + cli.main() + + +if __name__ == '__main__': + main() diff --git a/contrib/pydocstyle/checker.py b/contrib/pydocstyle/checker.py new file mode 100644 index 0000000..f6e039f --- /dev/null +++ b/contrib/pydocstyle/checker.py @@ -0,0 +1,719 @@ +"""Parsed source code checkers for docstring violations.""" + +import ast +import string +import sys +import tokenize as tk +from itertools import takewhile +from re import compile as re +from collections import namedtuple + +from . import violations +from .config import IllegalConfiguration +from .parser import (Package, Module, Class, NestedClass, Definition, AllError, + Method, Function, NestedFunction, Parser, StringIO, + ParseError) +from .utils import log, is_blank, pairwise +from .wordlists import IMPERATIVE_VERBS, IMPERATIVE_BLACKLIST, stem + + +__all__ = ('check', ) + + +# If possible (python >= 3.2) use tokenize.open to open files, so PEP 263 +# encoding markers are interpreted. +try: + tokenize_open = tk.open +except AttributeError: + tokenize_open = open + + +def check_for(kind, terminal=False): + def decorator(f): + f._check_for = kind + f._terminal = terminal + return f + return decorator + + +class ConventionChecker(object): + """Checker for PEP 257 and numpy conventions. + + D10x: Missing docstrings + D20x: Whitespace issues + D30x: Docstring formatting + D40x: Docstring content issues + + """ + + SECTION_NAMES = ['Short Summary', + 'Extended Summary', + 'Parameters', + 'Returns', + 'Yields', + 'Other Parameters', + 'Raises', + 'See Also', + 'Notes', + 'References', + 'Examples', + 'Attributes', + 'Methods'] + + def check_source(self, source, filename, ignore_decorators=None): + module = parse(StringIO(source), filename) + for definition in module: + for this_check in self.checks: + terminate = False + if isinstance(definition, this_check._check_for): + skipping_all = (definition.skipped_error_codes == 'all') + decorator_skip = ignore_decorators is not None and any( + len(ignore_decorators.findall(dec.name)) > 0 + for dec in definition.decorators) + if not skipping_all and not decorator_skip: + error = this_check(self, definition, + definition.docstring) + else: + error = None + errors = error if hasattr(error, '__iter__') else [error] + for error in errors: + if error is not None and error.code not in \ + definition.skipped_error_codes: + partition = this_check.__doc__.partition('.\n') + message, _, explanation = partition + error.set_context(explanation=explanation, + definition=definition) + yield error + if this_check._terminal: + terminate = True + break + if terminate: + break + + @property + def checks(self): + all = [this_check for this_check in vars(type(self)).values() + if hasattr(this_check, '_check_for')] + return sorted(all, key=lambda this_check: not this_check._terminal) + + @check_for(Definition, terminal=True) + def check_docstring_missing(self, definition, docstring): + """D10{0,1,2,3}: Public definitions should have docstrings. + + All modules should normally have docstrings. [...] all functions and + classes exported by a module should also have docstrings. Public + methods (including the __init__ constructor) should also have + docstrings. + + Note: Public (exported) definitions are either those with names listed + in __all__ variable (if present), or those that do not start + with a single underscore. + + """ + if (not docstring and definition.is_public or + docstring and is_blank(ast.literal_eval(docstring))): + codes = {Module: violations.D100, + Class: violations.D101, + NestedClass: violations.D106, + Method: (lambda: violations.D105() if definition.is_magic + else (violations.D107() if definition.is_init + else violations.D102())), + Function: violations.D103, + NestedFunction: violations.D103, + Package: violations.D104} + return codes[type(definition)]() + + @check_for(Definition) + def check_one_liners(self, definition, docstring): + """D200: One-liner docstrings should fit on one line with quotes. + + The closing quotes are on the same line as the opening quotes. + This looks better for one-liners. + + """ + if docstring: + lines = ast.literal_eval(docstring).split('\n') + if len(lines) > 1: + non_empty_lines = sum(1 for l in lines if not is_blank(l)) + if non_empty_lines == 1: + return violations.D200(len(lines)) + + @check_for(Function) + def check_no_blank_before(self, function, docstring): # def + """D20{1,2}: No blank lines allowed around function/method docstring. + + There's no blank line either before or after the docstring. + + """ + if docstring: + before, _, after = function.source.partition(docstring) + blanks_before = list(map(is_blank, before.split('\n')[:-1])) + blanks_after = list(map(is_blank, after.split('\n')[1:])) + blanks_before_count = sum(takewhile(bool, reversed(blanks_before))) + blanks_after_count = sum(takewhile(bool, blanks_after)) + if blanks_before_count != 0: + yield violations.D201(blanks_before_count) + if not all(blanks_after) and blanks_after_count != 0: + yield violations.D202(blanks_after_count) + + @check_for(Class) + def check_blank_before_after_class(self, class_, docstring): + """D20{3,4}: Class docstring should have 1 blank line around them. + + Insert a blank line before and after all docstrings (one-line or + multi-line) that document a class -- generally speaking, the class's + methods are separated from each other by a single blank line, and the + docstring needs to be offset from the first method by a blank line; + for symmetry, put a blank line between the class header and the + docstring. + + """ + # NOTE: this gives false-positive in this case + # class Foo: + # + # """Docstring.""" + # + # + # # comment here + # def foo(): pass + if docstring: + before, _, after = class_.source.partition(docstring) + blanks_before = list(map(is_blank, before.split('\n')[:-1])) + blanks_after = list(map(is_blank, after.split('\n')[1:])) + blanks_before_count = sum(takewhile(bool, reversed(blanks_before))) + blanks_after_count = sum(takewhile(bool, blanks_after)) + if blanks_before_count != 0: + yield violations.D211(blanks_before_count) + if blanks_before_count != 1: + yield violations.D203(blanks_before_count) + if not all(blanks_after) and blanks_after_count != 1: + yield violations.D204(blanks_after_count) + + @check_for(Definition) + def check_blank_after_summary(self, definition, docstring): + """D205: Put one blank line between summary line and description. + + Multi-line docstrings consist of a summary line just like a one-line + docstring, followed by a blank line, followed by a more elaborate + description. The summary line may be used by automatic indexing tools; + it is important that it fits on one line and is separated from the + rest of the docstring by a blank line. + + """ + if docstring: + lines = ast.literal_eval(docstring).strip().split('\n') + if len(lines) > 1: + post_summary_blanks = list(map(is_blank, lines[1:])) + blanks_count = sum(takewhile(bool, post_summary_blanks)) + if blanks_count != 1: + return violations.D205(blanks_count) + + @staticmethod + def _get_docstring_indent(definition, docstring): + """Return the indentation of the docstring's opening quotes.""" + before_docstring, _, _ = definition.source.partition(docstring) + _, _, indent = before_docstring.rpartition('\n') + return indent + + @check_for(Definition) + def check_indent(self, definition, docstring): + """D20{6,7,8}: The entire docstring should be indented same as code. + + The entire docstring is indented the same as the quotes at its + first line. + + """ + if docstring: + indent = self._get_docstring_indent(definition, docstring) + lines = docstring.split('\n') + if len(lines) > 1: + lines = lines[1:] # First line does not need indent. + indents = [leading_space(l) for l in lines if not is_blank(l)] + if set(' \t') == set(''.join(indents) + indent): + yield violations.D206() + if (len(indents) > 1 and min(indents[:-1]) > indent or + indents[-1] > indent): + yield violations.D208() + if min(indents) < indent: + yield violations.D207() + + @check_for(Definition) + def check_newline_after_last_paragraph(self, definition, docstring): + """D209: Put multi-line docstring closing quotes on separate line. + + Unless the entire docstring fits on a line, place the closing + quotes on a line by themselves. + + """ + if docstring: + lines = [l for l in ast.literal_eval(docstring).split('\n') + if not is_blank(l)] + if len(lines) > 1: + if docstring.split("\n")[-1].strip() not in ['"""', "'''"]: + return violations.D209() + + @check_for(Definition) + def check_surrounding_whitespaces(self, definition, docstring): + """D210: No whitespaces allowed surrounding docstring text.""" + if docstring: + lines = ast.literal_eval(docstring).split('\n') + if lines[0].startswith(' ') or \ + len(lines) == 1 and lines[0].endswith(' '): + return violations.D210() + + @check_for(Definition) + def check_multi_line_summary_start(self, definition, docstring): + """D21{2,3}: Multi-line docstring summary style check. + + A multi-line docstring summary should start either at the first, + or separately at the second line of a docstring. + + """ + if docstring: + start_triple = [ + '"""', "'''", + 'u"""', "u'''", + 'r"""', "r'''", + 'ur"""', "ur'''" + ] + + lines = ast.literal_eval(docstring).split('\n') + if len(lines) > 1: + first = docstring.split("\n")[0].strip().lower() + if first in start_triple: + return violations.D212() + else: + return violations.D213() + + @check_for(Definition) + def check_triple_double_quotes(self, definition, docstring): + r'''D300: Use """triple double quotes""". + + For consistency, always use """triple double quotes""" around + docstrings. Use r"""raw triple double quotes""" if you use any + backslashes in your docstrings. For Unicode docstrings, use + u"""Unicode triple-quoted strings""". + + Note: Exception to this is made if the docstring contains + """ quotes in its body. + + ''' + if docstring: + if '"""' in ast.literal_eval(docstring): + # Allow ''' quotes if docstring contains """, because + # otherwise """ quotes could not be expressed inside + # docstring. Not in PEP 257. + regex = re(r"[uU]?[rR]?'''[^'].*") + else: + regex = re(r'[uU]?[rR]?"""[^"].*') + + if not regex.match(docstring): + illegal_matcher = re(r"""[uU]?[rR]?("+|'+).*""") + illegal_quotes = illegal_matcher.match(docstring).group(1) + return violations.D300(illegal_quotes) + + @check_for(Definition) + def check_backslashes(self, definition, docstring): + r'''D301: Use r""" if any backslashes in a docstring. + + Use r"""raw triple double quotes""" if you use any backslashes + (\) in your docstrings. + + ''' + # Just check that docstring is raw, check_triple_double_quotes + # ensures the correct quotes. + if docstring and '\\' in docstring and not docstring.startswith( + ('r', 'ur')): + return violations.D301() + + @check_for(Definition) + def check_unicode_docstring(self, definition, docstring): + r'''D302: Use u""" for docstrings with Unicode. + + For Unicode docstrings, use u"""Unicode triple-quoted strings""". + + ''' + if 'unicode_literals' in definition.module.future_imports: + return + + # Just check that docstring is unicode, check_triple_double_quotes + # ensures the correct quotes. + if docstring and sys.version_info[0] <= 2: + if not is_ascii(docstring) and not docstring.startswith( + ('u', 'ur')): + return violations.D302() + + @check_for(Definition) + def check_ends_with_period(self, definition, docstring): + """D400: First line should end with a period. + + The [first line of a] docstring is a phrase ending in a period. + + """ + if docstring: + summary_line = ast.literal_eval(docstring).strip().split('\n')[0] + if not summary_line.endswith('.'): + return violations.D400(summary_line[-1]) + + @check_for(Function) + def check_imperative_mood(self, function, docstring): # def context + """D401: First line should be in imperative mood: 'Do', not 'Does'. + + [Docstring] prescribes the function or method's effect as a command: + ("Do this", "Return that"), not as a description; e.g. don't write + "Returns the pathname ...". + + """ + if docstring and not function.is_test: + stripped = ast.literal_eval(docstring).strip() + if stripped: + first_word = stripped.split()[0] + check_word = first_word.lower() + + if check_word in IMPERATIVE_BLACKLIST: + return violations.D401b(first_word) + + try: + correct_form = IMPERATIVE_VERBS.get(stem(check_word)) + except UnicodeDecodeError: + # This is raised when the docstring contains unicode + # characters in the first word, but is not a unicode + # string. In which case D302 will be reported. Ignoring. + return + + if correct_form and correct_form != check_word: + return violations.D401( + correct_form.capitalize(), + first_word + ) + + @check_for(Function) + def check_no_signature(self, function, docstring): # def context + """D402: First line should not be function's or method's "signature". + + The one-line docstring should NOT be a "signature" reiterating the + function/method parameters (which can be obtained by introspection). + + """ + if docstring: + first_line = ast.literal_eval(docstring).strip().split('\n')[0] + if function.name + '(' in first_line.replace(' ', ''): + return violations.D402() + + @check_for(Function) + def check_capitalized(self, function, docstring): + """D403: First word of the first line should be properly capitalized. + + The [first line of a] docstring is a phrase ending in a period. + + """ + if docstring: + first_word = ast.literal_eval(docstring).split()[0] + if first_word == first_word.upper(): + return + for char in first_word: + if char not in string.ascii_letters and char != "'": + return + if first_word != first_word.capitalize(): + return violations.D403(first_word.capitalize(), first_word) + + @check_for(Definition) + def check_starts_with_this(self, function, docstring): + """D404: First word of the docstring should not be `This`. + + Docstrings should use short, simple language. They should not begin + with "This class is [..]" or "This module contains [..]". + + """ + if docstring: + first_word = ast.literal_eval(docstring).split()[0] + if first_word.lower() == 'this': + return violations.D404() + + @staticmethod + def _get_leading_words(line): + """Return any leading set of words from `line`. + + For example, if `line` is " Hello world!!!", returns "Hello world". + """ + result = re("[A-Za-z ]+").match(line.strip()) + if result is not None: + return result.group() + + @staticmethod + def _is_a_docstring_section(context): + """Check if the suspected context is really a section header. + + Lets have a look at the following example docstring: + '''Title. + + Some part of the docstring that specifies what the function + returns. <----- Not a real section name. It has a suffix and the + previous line is not empty and does not end with + a punctuation sign. + + This is another line in the docstring. It describes stuff, + but we forgot to add a blank line between it and the section name. + Returns <----- A real section name. The previous line ends with + ------- a period, therefore it is in a new + grammatical context. + Bla. + + ''' + + To make sure this is really a section we check these conditions: + * There's no suffix to the section name. + * The previous line ends with punctuation. + * The previous line is empty. + + If one of the conditions is true, we will consider the line as + a section name. + """ + section_name_suffix = context.line.lstrip(context.section_name).strip() + + punctuation = [',', ';', '.', '-', '\\', '/', ']', '}', ')'] + prev_line_ends_with_punctuation = \ + any(context.previous_line.strip().endswith(x) for x in punctuation) + + return (is_blank(section_name_suffix) or + prev_line_ends_with_punctuation or + is_blank(context.previous_line)) + + @classmethod + def _check_section_underline(cls, section_name, context, indentation): + """D4{07,08,09,12}, D215: Section underline checks. + + Check for correct formatting for docstring sections. Checks that: + * The line that follows the section name contains + dashes (D40{7,8}). + * The amount of dashes is equal to the length of the section + name (D409). + * The section's content does not begin in the line that follows + the section header (D412). + * The indentation of the dashed line is equal to the docstring's + indentation (D215). + """ + blank_lines_after_header = 0 + + for line in context.following_lines: + if not is_blank(line): + break + blank_lines_after_header += 1 + else: + # There are only blank lines after the header. + yield violations.D407(section_name) + return + + non_empty_line = context.following_lines[blank_lines_after_header] + dash_line_found = ''.join(set(non_empty_line.strip())) == '-' + + if not dash_line_found: + yield violations.D407(section_name) + if blank_lines_after_header > 0: + yield violations.D412(section_name) + else: + if blank_lines_after_header > 0: + yield violations.D408(section_name) + + if non_empty_line.strip() != "-" * len(section_name): + yield violations.D409(len(section_name), + section_name, + len(non_empty_line.strip())) + + if leading_space(non_empty_line) > indentation: + yield violations.D215(section_name) + + line_after_dashes_index = blank_lines_after_header + 1 + # If the line index after the dashes is in range (perhaps we have + # a header + underline followed by another section header). + if line_after_dashes_index < len(context.following_lines): + line_after_dashes = \ + context.following_lines[line_after_dashes_index] + if is_blank(line_after_dashes): + rest_of_lines = \ + context.following_lines[line_after_dashes_index:] + if not is_blank(''.join(rest_of_lines)): + yield violations.D412(section_name) + else: + yield violations.D414(section_name) + else: + yield violations.D414(section_name) + + @classmethod + def _check_section(cls, docstring, definition, context): + """D4{05,06,10,11,13}, D214: Section name checks. + + Check for valid section names. Checks that: + * The section name is properly capitalized (D405). + * The section is not over-indented (D214). + * The section name has no superfluous suffix to it (D406). + * There's a blank line after the section (D410, D413). + * There's a blank line before the section (D411). + + Also yields all the errors from `_check_section_underline`. + """ + capitalized_section = context.section_name.title() + indentation = cls._get_docstring_indent(definition, docstring) + + if (context.section_name not in cls.SECTION_NAMES and + capitalized_section in cls.SECTION_NAMES): + yield violations.D405(capitalized_section, context.section_name) + + if leading_space(context.line) > indentation: + yield violations.D214(capitalized_section) + + suffix = context.line.strip().lstrip(context.section_name) + if suffix: + yield violations.D406(capitalized_section, context.line.strip()) + + if (not context.following_lines or + not is_blank(context.following_lines[-1])): + if context.is_last_section: + yield violations.D413(capitalized_section) + else: + yield violations.D410(capitalized_section) + + if not is_blank(context.previous_line): + yield violations.D411(capitalized_section) + + for err in cls._check_section_underline(capitalized_section, + context, + indentation): + yield err + + @check_for(Definition) + def check_docstring_sections(self, definition, docstring): + """D21{4,5}, D4{05,06,07,08,09,10}: Docstring sections checks. + + Check the general format of a sectioned docstring: + '''This is my one-liner. + + Short Summary + ------------- + This is my summary. + + Returns + ------- + None. + + ''' + + Section names appear in `SECTION_NAMES`. + """ + if not docstring: + return + + lines = docstring.split("\n") + if len(lines) < 2: + return + + lower_section_names = [s.lower() for s in self.SECTION_NAMES] + + def _suspected_as_section(_line): + result = self._get_leading_words(_line.lower()) + return result in lower_section_names + + # Finding our suspects. + suspected_section_indices = [i for i, line in enumerate(lines) if + _suspected_as_section(line)] + + SectionContext = namedtuple('SectionContext', ('section_name', + 'previous_line', + 'line', + 'following_lines', + 'original_index', + 'is_last_section')) + + # First - create a list of possible contexts. Note that the + # `following_linex` member is until the end of the docstring. + contexts = (SectionContext(self._get_leading_words(lines[i].strip()), + lines[i - 1], + lines[i], + lines[i + 1:], + i, + False) + for i in suspected_section_indices) + + # Now that we have manageable objects - rule out false positives. + contexts = (c for c in contexts if self._is_a_docstring_section(c)) + + # Now we shall trim the `following lines` field to only reach the + # next section name. + for a, b in pairwise(contexts, None): + end = -1 if b is None else b.original_index + new_ctx = SectionContext(a.section_name, + a.previous_line, + a.line, + lines[a.original_index + 1:end], + a.original_index, + b is None) + for err in self._check_section(docstring, definition, new_ctx): + yield err + + +parse = Parser() + + +def check(filenames, select=None, ignore=None, ignore_decorators=None): + """Generate docstring errors that exist in `filenames` iterable. + + By default, the PEP-257 convention is checked. To specifically define the + set of error codes to check for, supply either `select` or `ignore` (but + not both). In either case, the parameter should be a collection of error + code strings, e.g., {'D100', 'D404'}. + + When supplying `select`, only specified error codes will be reported. + When supplying `ignore`, all error codes which were not specified will be + reported. + + Note that ignored error code refer to the entire set of possible + error codes, which is larger than just the PEP-257 convention. To your + convenience, you may use `pydocstyle.violations.conventions.pep257` as + a base set to add or remove errors from. + + Examples + --------- + >>> check(['pydocstyle.py']) + + + >>> check(['pydocstyle.py'], select=['D100']) + + + >>> check(['pydocstyle.py'], ignore=conventions.pep257 - {'D100'}) + + + """ + if select is not None and ignore is not None: + raise IllegalConfiguration('Cannot pass both select and ignore. ' + 'They are mutually exclusive.') + elif select is not None: + checked_codes = select + elif ignore is not None: + checked_codes = list(set(violations.ErrorRegistry.get_error_codes()) - + set(ignore)) + else: + checked_codes = violations.conventions.pep257 + + for filename in filenames: + log.info('Checking file %s.', filename) + try: + with tokenize_open(filename) as file: + source = file.read() + for error in ConventionChecker().check_source(source, filename, + ignore_decorators): + code = getattr(error, 'code', None) + if code in checked_codes: + yield error + except (EnvironmentError, AllError, ParseError) as error: + log.warning('Error in file %s: %s', filename, error) + yield error + except tk.TokenError: + yield SyntaxError('invalid syntax in file %s' % filename) + + +def is_ascii(string): + return all(ord(char) < 128 for char in string) + + +def leading_space(string): + return re('\s*').match(string).group() diff --git a/contrib/pydocstyle/cli.py b/contrib/pydocstyle/cli.py new file mode 100644 index 0000000..c195502 --- /dev/null +++ b/contrib/pydocstyle/cli.py @@ -0,0 +1,96 @@ +"""Command line interface for pydocstyle.""" +import logging +import sys + +from .utils import log +from .violations import Error +from .config import ConfigurationParser, IllegalConfiguration +from .checker import check + + +__all__ = ('main', ) + + +class ReturnCode(object): + no_violations_found = 0 + violations_found = 1 + invalid_options = 2 + + +def run_pydocstyle(): + log.setLevel(logging.DEBUG) + conf = ConfigurationParser() + setup_stream_handlers(conf.get_default_run_configuration()) + + try: + conf.parse() + except IllegalConfiguration: + return ReturnCode.invalid_options + + run_conf = conf.get_user_run_configuration() + + # Reset the logger according to the command line arguments + setup_stream_handlers(run_conf) + + log.debug("starting in debug mode.") + + Error.explain = run_conf.explain + Error.source = run_conf.source + + errors = [] + try: + for filename, checked_codes, ignore_decorators in \ + conf.get_files_to_check(): + errors.extend(check((filename,), select=checked_codes, + ignore_decorators=ignore_decorators)) + except IllegalConfiguration as error: + # An illegal configuration file was found during file generation. + log.error(error.args[0]) + return ReturnCode.invalid_options + + count = 0 + for error in errors: + if hasattr(error, 'code'): + sys.stdout.write('%s\n' % error) + count += 1 + if count == 0: + exit_code = ReturnCode.no_violations_found + else: + exit_code = ReturnCode.violations_found + if run_conf.count: + print(count) + return exit_code + + +def main(): + """Run pydocstyle as a script.""" + try: + sys.exit(run_pydocstyle()) + except KeyboardInterrupt: + pass + + +def setup_stream_handlers(conf): + """Setup logging stream handlers according to the options.""" + class StdoutFilter(logging.Filter): + def filter(self, record): + return record.levelno in (logging.DEBUG, logging.INFO) + + log.handlers = [] + + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.WARNING) + stdout_handler.addFilter(StdoutFilter()) + if conf.debug: + stdout_handler.setLevel(logging.DEBUG) + elif conf.verbose: + stdout_handler.setLevel(logging.INFO) + else: + stdout_handler.setLevel(logging.WARNING) + log.addHandler(stdout_handler) + + stderr_handler = logging.StreamHandler(sys.stderr) + msg_format = "%(levelname)s: %(message)s" + stderr_handler.setFormatter(logging.Formatter(fmt=msg_format)) + stderr_handler.setLevel(logging.WARNING) + log.addHandler(stderr_handler) diff --git a/contrib/pydocstyle/config.py b/contrib/pydocstyle/config.py new file mode 100644 index 0000000..3f4d915 --- /dev/null +++ b/contrib/pydocstyle/config.py @@ -0,0 +1,647 @@ +"""Configuration file parsing and utilities.""" + +import copy +import itertools +import os +from collections import Set, namedtuple +from re import compile as re + + +from configparser import RawConfigParser + + +from .utils import __version__, log +from .violations import ErrorRegistry, conventions + + +def check_initialized(method): + """Check that the configuration object was initialized.""" + def _decorator(self, *args, **kwargs): + if self._arguments is None or self._options is None: + raise RuntimeError('using an uninitialized configuration') + return method(self, *args, **kwargs) + return _decorator + + +class ConfigurationParser(object): + """Responsible for parsing configuration from files and CLI. + + There are 2 types of configurations: Run configurations and Check + configurations. + + Run Configurations: + ------------------ + Responsible for deciding things that are related to the user interface and + configuration discovery, e.g. verbosity, debug options, etc. + All run configurations default to `False` or `None` and are decided only + by CLI. + + Check Configurations: + -------------------- + Configurations that are related to which files and errors will be checked. + These are configurable in 2 ways: using the CLI, and using configuration + files. + + Configuration files are nested within the file system, meaning that the + closer a configuration file is to a checked file, the more relevant it will + be. For instance, imagine this directory structure: + + A + +-- tox.ini: sets `select=D100` + +-- B + +-- foo.py + +-- tox.ini: sets `add-ignore=D100` + + Then `foo.py` will not be checked for `D100`. + The configuration build algorithm is described in `self._get_config`. + + Note: If any of `BASE_ERROR_SELECTION_OPTIONS` was selected in the CLI, all + configuration files will be ignored and each file will be checked for + the error codes supplied in the CLI. + + """ + + CONFIG_FILE_OPTIONS = ('convention', 'select', 'ignore', 'add-select', + 'add-ignore', 'match', 'match-dir', + 'ignore-decorators') + BASE_ERROR_SELECTION_OPTIONS = ('ignore', 'select', 'convention') + + DEFAULT_MATCH_RE = '(?!test_).*\.py' + DEFAULT_MATCH_DIR_RE = '[^\.].*' + DEFAULT_IGNORE_DECORATORS_RE = '' + DEFAULT_CONVENTION = conventions.pep257 + + PROJECT_CONFIG_FILES = ( + 'setup.cfg', + 'tox.ini', + '.pydocstyle', + '.pydocstyle.ini', + '.pydocstylerc', + '.pydocstylerc.ini', + # The following is deprecated, but remains for backwards compatibility. + '.pep257', + ) + + POSSIBLE_SECTION_NAMES = ('pydocstyle', 'pep257') + + def __init__(self): + """Create a configuration parser.""" + self._cache = {} + self._override_by_cli = None + self._options = self._arguments = self._run_conf = None + self._parser = self._create_option_parser() + + # ---------------------------- Public Methods ----------------------------- + + def get_default_run_configuration(self): + """Return a `RunConfiguration` object set with default values.""" + options, _ = self._parse_args([]) + return self._create_run_config(options) + + def parse(self): + """Parse the configuration. + + If one of `BASE_ERROR_SELECTION_OPTIONS` was selected, overrides all + error codes to check and disregards any error code related + configurations from the configuration files. + + """ + self._options, self._arguments = self._parse_args() + self._arguments = self._arguments or ['.'] + + if not self._validate_options(self._options): + raise IllegalConfiguration() + + self._run_conf = self._create_run_config(self._options) + + config = self._create_check_config(self._options, use_defaults=False) + self._override_by_cli = config + + @check_initialized + def get_user_run_configuration(self): + """Return the run configuration for the script.""" + return self._run_conf + + @check_initialized + def get_files_to_check(self): + """Generate files and error codes to check on each one. + + Walk dir trees under `self._arguments` and yield file names + that `match` under each directory that `match_dir`. + The method locates the configuration for each file name and yields a + tuple of (filename, [error_codes]). + + With every discovery of a new configuration file `IllegalConfiguration` + might be raised. + + """ + def _get_matches(config): + """Return the `match` and `match_dir` functions for `config`.""" + match_func = re(config.match + '$').match + match_dir_func = re(config.match_dir + '$').match + return match_func, match_dir_func + + def _get_ignore_decorators(config): + """Return the `ignore_decorators` as None or regex.""" + if config.ignore_decorators: # not None and not '' + ignore_decorators = re(config.ignore_decorators) + else: + ignore_decorators = None + return ignore_decorators + + for name in self._arguments: + if os.path.isdir(name): + for root, dirs, filenames in os.walk(name): + config = self._get_config(os.path.abspath(root)) + match, match_dir = _get_matches(config) + ignore_decorators = _get_ignore_decorators(config) + + # Skip any dirs that do not match match_dir + dirs[:] = [dir for dir in dirs if match_dir(dir)] + + for filename in filenames: + if match(filename): + full_path = os.path.join(root, filename) + yield (full_path, list(config.checked_codes), + ignore_decorators) + else: + config = self._get_config(os.path.abspath(name)) + match, _ = _get_matches(config) + ignore_decorators = _get_ignore_decorators(config) + if match(name): + yield (name, list(config.checked_codes), ignore_decorators) + + # --------------------------- Private Methods ----------------------------- + + def _get_config_by_discovery(self, node): + """Get a configuration for checking `node` by config discovery. + + Config discovery happens when no explicit config file is specified. The + file system is searched for config files starting from the directory + containing the file being checked, and up until the root directory of + the project. + + See `_get_config` for further details. + + """ + path = self._get_node_dir(node) + + if path in self._cache: + return self._cache[path] + + config_file = self._get_config_file_in_folder(path) + + if config_file is None: + parent_dir, tail = os.path.split(path) + if tail: + # No configuration file, simply take the parent's. + config = self._get_config(parent_dir) + else: + # There's no configuration file and no parent directory. + # Use the default configuration or the one given in the CLI. + config = self._create_check_config(self._options) + else: + # There's a config file! Read it and merge if necessary. + options, inherit = self._read_configuration_file(config_file) + + parent_dir, tail = os.path.split(path) + if tail and inherit: + # There is a parent dir and we should try to merge. + parent_config = self._get_config(parent_dir) + config = self._merge_configuration(parent_config, options) + else: + # No need to merge or parent dir does not exist. + config = self._create_check_config(options) + + return config + + def _get_config(self, node): + """Get and cache the run configuration for `node`. + + If no configuration exists (not local and not for the parent node), + returns and caches a default configuration. + + The algorithm: + ------------- + * If the current directory's configuration exists in + `self._cache` - return it. + * If a configuration file does not exist in this directory: + * If the directory is not a root directory: + * Cache its configuration as this directory's and return it. + * Else: + * Cache a default configuration and return it. + * Else: + * Read the configuration file. + * If a parent directory exists AND the configuration file + allows inheritance: + * Read the parent configuration by calling this function with the + parent directory as `node`. + * Merge the parent configuration with the current one and + cache it. + * If the user has specified one of `BASE_ERROR_SELECTION_OPTIONS` in + the CLI - return the CLI configuration with the configuration match + clauses + * Set the `--add-select` and `--add-ignore` CLI configurations. + + """ + if self._run_conf.config is None: + log.debug('No config file specified, discovering.') + config = self._get_config_by_discovery(node) + else: + log.debug('Using config file %r', self._run_conf.config) + if not os.path.exists(self._run_conf.config): + raise IllegalConfiguration('Configuration file {!r} specified ' + 'via --config was not found.' + .format(self._run_conf.config)) + + if None in self._cache: + return self._cache[None] + options, _ = self._read_configuration_file(self._run_conf.config) + + if options is None: + log.warning('Configuration file does not contain a ' + 'pydocstyle section. Using default configuration.') + config = self._create_check_config(self._options) + else: + config = self._create_check_config(options) + + # Make the CLI always win + final_config = {} + for attr in CheckConfiguration._fields: + cli_val = getattr(self._override_by_cli, attr) + conf_val = getattr(config, attr) + final_config[attr] = cli_val if cli_val is not None else conf_val + + config = CheckConfiguration(**final_config) + + self._set_add_options(config.checked_codes, self._options) + + # Handle caching + if self._run_conf.config is not None: + self._cache[None] = config + else: + self._cache[self._get_node_dir(node)] = config + return config + + @staticmethod + def _get_node_dir(node): + """Return the absolute path of the directory of a filesystem node.""" + path = os.path.abspath(node) + return path if os.path.isdir(path) else os.path.dirname(path) + + def _read_configuration_file(self, path): + """Try to read and parse `path` as a configuration file. + + If the configurations were illegal (checked with + `self._validate_options`), raises `IllegalConfiguration`. + + Returns (options, should_inherit). + + """ + parser = RawConfigParser(inline_comment_prefixes=('#', ';')) + options = None + should_inherit = True + + if parser.read(path) and self._get_section_name(parser): + all_options = self._parser.option_list[:] + for group in self._parser.option_groups: + all_options.extend(group.option_list) + + option_list = dict([(o.dest, o.type or o.action) + for o in all_options]) + + # First, read the default values + new_options, _ = self._parse_args([]) + + # Second, parse the configuration + section_name = self._get_section_name(parser) + for opt in parser.options(section_name): + if opt == 'inherit': + should_inherit = parser.getboolean(section_name, opt) + continue + + if opt.replace('_', '-') not in self.CONFIG_FILE_OPTIONS: + log.warning("Unknown option '{}' ignored".format(opt)) + continue + + normalized_opt = opt.replace('-', '_') + opt_type = option_list[normalized_opt] + if opt_type in ('int', 'count'): + value = parser.getint(section_name, opt) + elif opt_type == 'string': + value = parser.get(section_name, opt) + else: + assert opt_type in ('store_true', 'store_false') + value = parser.getboolean(section_name, opt) + setattr(new_options, normalized_opt, value) + + # Third, fix the set-options + options = self._fix_set_options(new_options) + + if options is not None: + if not self._validate_options(options): + raise IllegalConfiguration('in file: {}'.format(path)) + + return options, should_inherit + + def _merge_configuration(self, parent_config, child_options): + """Merge parent config into the child options. + + The migration process requires an `options` object for the child in + order to distinguish between mutually exclusive codes, add-select and + add-ignore error codes. + + """ + # Copy the parent error codes so we won't override them + error_codes = copy.deepcopy(parent_config.checked_codes) + if self._has_exclusive_option(child_options): + error_codes = self._get_exclusive_error_codes(child_options) + + self._set_add_options(error_codes, child_options) + + kwargs = dict(checked_codes=error_codes) + for key in ('match', 'match_dir', 'ignore_decorators'): + kwargs[key] = \ + getattr(child_options, key) or getattr(parent_config, key) + return CheckConfiguration(**kwargs) + + def _parse_args(self, args=None, values=None): + """Parse the options using `self._parser` and reformat the options.""" + options, arguments = self._parser.parse_args(args, values) + return self._fix_set_options(options), arguments + + @staticmethod + def _create_run_config(options): + """Create a `RunConfiguration` object from `options`.""" + values = dict([(opt, getattr(options, opt)) for opt in + RunConfiguration._fields]) + return RunConfiguration(**values) + + @classmethod + def _create_check_config(cls, options, use_defaults=True): + """Create a `CheckConfiguration` object from `options`. + + If `use_defaults`, any of the match options that are `None` will + be replaced with their default value and the default convention will be + set for the checked codes. + + """ + checked_codes = None + + if cls._has_exclusive_option(options) or use_defaults: + checked_codes = cls._get_checked_errors(options) + + kwargs = dict(checked_codes=checked_codes) + for key in ('match', 'match_dir', 'ignore_decorators'): + kwargs[key] = getattr(cls, 'DEFAULT_{0}_RE'.format(key.upper())) \ + if getattr(options, key) is None and use_defaults \ + else getattr(options, key) + return CheckConfiguration(**kwargs) + + @classmethod + def _get_section_name(cls, parser): + """Parse options from relevant section.""" + for section_name in cls.POSSIBLE_SECTION_NAMES: + if parser.has_section(section_name): + return section_name + + return None + + @classmethod + def _get_config_file_in_folder(cls, path): + """Look for a configuration file in `path`. + + If exists return its full path, otherwise None. + + """ + if os.path.isfile(path): + path = os.path.dirname(path) + + for fn in cls.PROJECT_CONFIG_FILES: + config = RawConfigParser() + full_path = os.path.join(path, fn) + if config.read(full_path) and cls._get_section_name(config): + return full_path + + @classmethod + def _get_exclusive_error_codes(cls, options): + """Extract the error codes from the selected exclusive option.""" + codes = set(ErrorRegistry.get_error_codes()) + checked_codes = None + + if options.ignore is not None: + ignored = cls._expand_error_codes(options.ignore) + checked_codes = codes - ignored + elif options.select is not None: + checked_codes = cls._expand_error_codes(options.select) + elif options.convention is not None: + checked_codes = getattr(conventions, options.convention) + + # To not override the conventions nor the options - copy them. + return copy.deepcopy(checked_codes) + + @classmethod + def _set_add_options(cls, checked_codes, options): + """Set `checked_codes` by the `add_ignore` or `add_select` options.""" + checked_codes |= cls._expand_error_codes(options.add_select) + checked_codes -= cls._expand_error_codes(options.add_ignore) + + @staticmethod + def _expand_error_codes(code_parts): + """Return an expanded set of error codes to ignore.""" + codes = set(ErrorRegistry.get_error_codes()) + expanded_codes = set() + + try: + for part in code_parts: + # Dealing with split-lined configurations; The part might begin + # with a whitespace due to the newline character. + part = part.strip() + if not part: + continue + + codes_to_add = {code for code in codes + if code.startswith(part)} + if not codes_to_add: + log.warn('Error code passed is not a prefix of any known ' + 'errors: %s', part) + expanded_codes.update(codes_to_add) + except TypeError as e: + raise IllegalConfiguration(e) + + return expanded_codes + + @classmethod + def _get_checked_errors(cls, options): + """Extract the codes needed to be checked from `options`.""" + checked_codes = cls._get_exclusive_error_codes(options) + if checked_codes is None: + checked_codes = cls.DEFAULT_CONVENTION + + cls._set_add_options(checked_codes, options) + + return checked_codes + + @classmethod + def _validate_options(cls, options): + """Validate the mutually exclusive options. + + Return `True` iff only zero or one of `BASE_ERROR_SELECTION_OPTIONS` + was selected. + + """ + for opt1, opt2 in \ + itertools.permutations(cls.BASE_ERROR_SELECTION_OPTIONS, 2): + if getattr(options, opt1) and getattr(options, opt2): + log.error('Cannot pass both {} and {}. They are ' + 'mutually exclusive.'.format(opt1, opt2)) + return False + + if options.convention and options.convention not in conventions: + log.error("Illegal convention '{}'. Possible conventions: {}" + .format(options.convention, + ', '.join(conventions.keys()))) + return False + return True + + @classmethod + def _has_exclusive_option(cls, options): + """Return `True` iff one or more exclusive options were selected.""" + return any([getattr(options, opt) is not None for opt in + cls.BASE_ERROR_SELECTION_OPTIONS]) + + @classmethod + def _fix_set_options(cls, options): + """Alter the set options from None/strings to sets in place.""" + optional_set_options = ('ignore', 'select') + mandatory_set_options = ('add_ignore', 'add_select') + + def _get_set(value_str): + """Split `value_str` by the delimiter `,` and return a set. + + Removes any occurrences of '' in the set. + Also expand error code prefixes, to avoid doing this for every + file. + + """ + return cls._expand_error_codes(set(value_str.split(',')) - {''}) + + for opt in optional_set_options: + value = getattr(options, opt) + if value is not None: + setattr(options, opt, _get_set(value)) + + for opt in mandatory_set_options: + value = getattr(options, opt) + if value is None: + value = '' + + if not isinstance(value, Set): + value = _get_set(value) + + setattr(options, opt, value) + + return options + + @classmethod + def _create_option_parser(cls): + """Return an option parser to parse the command line arguments.""" + from optparse import OptionParser, OptionGroup + + parser = OptionParser( + version=__version__, + usage='Usage: pydocstyle [options] [...]') + + option = parser.add_option + + # Run configuration options + option('-e', '--explain', action='store_true', default=False, + help='show explanation of each error') + option('-s', '--source', action='store_true', default=False, + help='show source for each error') + option('-d', '--debug', action='store_true', default=False, + help='print debug information') + option('-v', '--verbose', action='store_true', default=False, + help='print status information') + option('--count', action='store_true', default=False, + help='print total number of errors to stdout') + option('--config', metavar='', default=None, + help='use given config file and disable config discovery') + + check_group = OptionGroup( + parser, + 'Error Check Options', + 'Only one of --select, --ignore or --convention can be ' + 'specified. If none is specified, defaults to ' + '`--convention=pep257`. These three options select the "basic ' + 'list" of error codes to check. If you wish to change that list ' + '(for example, if you selected a known convention but wish to ' + 'ignore a specific error from it or add a new one) you can ' + 'use `--add-[ignore/select]` in order to do so.') + add_check = check_group.add_option + + # Error check options + add_check('--select', metavar='', default=None, + help='choose the basic list of checked errors by ' + 'specifying which errors to check for (with a list of ' + 'comma-separated error codes or prefixes). ' + 'for example: --select=D101,D2') + add_check('--ignore', metavar='', default=None, + help='choose the basic list of checked errors by ' + 'specifying which errors to ignore out of all of the ' + 'available error codes (with a list of ' + 'comma-separated error codes or prefixes). ' + 'for example: --ignore=D101,D2') + add_check('--convention', metavar='', default=None, + help='choose the basic list of checked errors by specifying ' + 'an existing convention. Possible conventions: {}.' + .format(', '.join(conventions))) + add_check('--add-select', metavar='', default=None, + help='add extra error codes to check to the basic list of ' + 'errors previously set by --select, --ignore or ' + '--convention.') + add_check('--add-ignore', metavar='', default=None, + help='ignore extra error codes by removing them from the ' + 'basic list previously set by --select, --ignore ' + 'or --convention.') + + parser.add_option_group(check_group) + + # Match clauses + option('--match', metavar='', default=None, + help=("check only files that exactly match regular " + "expression; default is --match='{}' which matches " + "files that don't start with 'test_' but end with " + "'.py'").format(cls.DEFAULT_MATCH_RE)) + option('--match-dir', metavar='', default=None, + help=("search only dirs that exactly match regular " + "expression; default is --match-dir='{}', which " + "matches all dirs that don't start with " + "a dot").format(cls.DEFAULT_MATCH_DIR_RE)) + + # Decorators + option('--ignore-decorators', metavar='', default=None, + help=("ignore any functions or methods that are decorated " + "by a function with a name fitting the " + "regular expression; default is --ignore-decorators='{0}'" + " which does not ignore any decorated functions." + .format(cls.DEFAULT_IGNORE_DECORATORS_RE))) + return parser + + +# Check configuration - used by the ConfigurationParser class. +CheckConfiguration = namedtuple('CheckConfiguration', + ('checked_codes', 'match', 'match_dir', + 'ignore_decorators')) + + +class IllegalConfiguration(Exception): + """An exception for illegal configurations.""" + + pass + + +# General configurations for pydocstyle run. +RunConfiguration = namedtuple('RunConfiguration', + ('explain', 'source', 'debug', + 'verbose', 'count', 'config')) diff --git a/contrib/pydocstyle/data/imperatives.txt b/contrib/pydocstyle/data/imperatives.txt new file mode 100644 index 0000000..5ff952f --- /dev/null +++ b/contrib/pydocstyle/data/imperatives.txt @@ -0,0 +1,232 @@ +# Imperative forms of verbs +# +# This file contains the imperative form of frequently encountered +# docstring verbs. Some of these may be more commonly encountered as +# nouns, but blacklisting them for this may cause false positives. +accept +access +add +adjust +aggregate +allow +append +apply +archive +assert +assign +attempt +authenticate +authorize +break +build +cache +calculate +call +cancel +capture +change +check +clean +clear +close +collect +combine +commit +compare +compute +configure +confirm +connect +construct +control +convert +copy +count +create +customize +declare +decode +decorate +define +delegate +delete +deprecate +derive +describe +detect +determine +display +download +drop +dump +emit +empty +enable +encapsulate +encode +end +ensure +enumerate +establish +evaluate +examine +execute +exit +expand +expect +export +extend +extract +feed +fetch +fill +filter +finalize +find +fire +fix +flag +force +format +forward +generate +get +give +go +group +handle +help +hold +identify +implement +import +indicate +init +initalise +initialise +initialize +input +insert +instantiate +intercept +invoke +iterate +join +keep +launch +list +listen +load +log +look +make +manage +manipulate +map +mark +match +merge +mock +modify +monitor +move +normalize +note +obtain +open +output +override +overwrite +pad +parse +partial +pass +perform +persist +pick +plot +poll +populate +post +prepare +print +process +produce +provide +publish +pull +put +query +raise +read +record +refer +refresh +register +reload +remove +rename +render +replace +reply +report +represent +request +require +reset +resolve +retrieve +return +roll +rollback +round +run +sample +save +scan +search +select +send +serialise +serialize +serve +set +show +simulate +source +specify +split +start +step +stop +store +strip +submit +subscribe +sum +swap +sync +synchronise +synchronize +take +tear +test +time +transform +translate +transmit +truncate +try +turn +tweak +update +upload +use +validate +verify +view +wait +walk +wrap +write +yield diff --git a/contrib/pydocstyle/data/imperatives_blacklist.txt b/contrib/pydocstyle/data/imperatives_blacklist.txt new file mode 100644 index 0000000..65018e8 --- /dev/null +++ b/contrib/pydocstyle/data/imperatives_blacklist.txt @@ -0,0 +1,100 @@ +# Blacklisted imperative words +# +# These are words that, if they begin a docstring, are a good indicator that +# the docstring is not written in an imperative voice. +# +# The words included in this list fall into a number of categories: +# +# - Starting with a noun/pronoun indicates that the docstring is a noun phrase +# or a sentence but not in the imperative mood +# - Adjectives are always followed by a noun, so same +# - Particles are also followed by a noun +# - Some adverbs don't really indicate an imperative sentence, for example +# "importantly" or "currently". +# - Some irregular verb forms that don't stem to the same string as the +# imperative does (eg. 'does') +a +an +the +action +always +api +base +basic +business +calculation +callback +collection +common +constructor +convenience +convenient +current +currently +custom +data +data +default +deprecated +description +dict +dictionary +does +dummy +example +factory +false +final +formula +function +generic +handler +handler +helper +here +hook +implementation +importantly +internal +it +main +method +module +new +number +optional +package +placeholder +reference +result +same +schema +setup +should +simple +some +special +sql +standard +static +string +subclasses +that +these +this +true +unique +unit +utility +what +wrapper + + +# These are nouns, but often used in the context of functions that act as +# objects; thus we do not blacklist these. +# +# context # as in context manager +# decorator +# class # as in class decorator +# property +# generator diff --git a/contrib/pydocstyle/parser.py b/contrib/pydocstyle/parser.py new file mode 100644 index 0000000..62ac4f5 --- /dev/null +++ b/contrib/pydocstyle/parser.py @@ -0,0 +1,599 @@ +"""Python code parser.""" + +import logging +import six +import textwrap +import tokenize as tk +from itertools import chain, dropwhile +from re import compile as re +from .utils import log + +try: + from StringIO import StringIO +except ImportError: # Python 3.0 and later + from io import StringIO + +try: + next +except NameError: # Python 2.5 and earlier + nothing = object() + + def next(obj, default=nothing): + if default == nothing: + return obj.next() + else: + try: + return obj.next() + except StopIteration: + return default + + +__all__ = ('Parser', 'Definition', 'Module', 'Package', 'Function', + 'NestedFunction', 'Method', 'Class', 'NestedClass', 'AllError', + 'StringIO', 'ParseError') + + +class ParseError(Exception): + def __str__(self): + return "Cannot parse file." + + +def humanize(string): + return re(r'(.)([A-Z]+)').sub(r'\1 \2', string).lower() + + +class Value(object): + """A generic object with a list of preset fields.""" + + def __init__(self, *args): + if len(self._fields) != len(args): + raise ValueError('got {} arguments for {} fields for {}: {}' + .format(len(args), len(self._fields), + self.__class__.__name__, self._fields)) + vars(self).update(zip(self._fields, args)) + + def __hash__(self): + return hash(repr(self)) + + def __eq__(self, other): + return other and vars(self) == vars(other) + + def __repr__(self): + kwargs = ', '.join('{}={!r}'.format(field, getattr(self, field)) + for field in self._fields) + return '{}({})'.format(self.__class__.__name__, kwargs) + + +class Definition(Value): + """A Python source code definition (could be class, function, etc).""" + + _fields = ('name', '_source', 'start', 'end', 'decorators', 'docstring', + 'children', 'parent', 'skipped_error_codes') + + _human = property(lambda self: humanize(type(self).__name__)) + kind = property(lambda self: self._human.split()[-1]) + module = property(lambda self: self.parent.module) + all = property(lambda self: self.module.all) + _slice = property(lambda self: slice(self.start - 1, self.end)) + is_class = False + + def __iter__(self): + return chain([self], *self.children) + + @property + def _publicity(self): + return {True: 'public', False: 'private'}[self.is_public] + + @property + def source(self): + """Return the source code for the definition.""" + full_src = self._source[self._slice] + + def is_empty_or_comment(line): + return line.strip() == '' or line.strip().startswith('#') + + filtered_src = dropwhile(is_empty_or_comment, reversed(full_src)) + return ''.join(reversed(list(filtered_src))) + + def __str__(self): + out = 'in {} {} `{}`'.format(self._publicity, self._human, self.name) + if self.skipped_error_codes: + out += ' (skipping {})'.format(self.skipped_error_codes) + return out + + +class Module(Definition): + """A Python source code module.""" + + _fields = ('name', '_source', 'start', 'end', 'decorators', 'docstring', + 'children', 'parent', '_all', 'future_imports', + 'skipped_error_codes') + _nest = staticmethod(lambda s: {'def': Function, 'class': Class}[s]) + module = property(lambda self: self) + all = property(lambda self: self._all) + + @property + def is_public(self): + return not self.name.startswith('_') or self.name.startswith('__') + + def __str__(self): + return 'at module level' + + +class Package(Module): + """A package is a __init__.py module.""" + + +class Function(Definition): + """A Python source code function.""" + + _nest = staticmethod(lambda s: {'def': NestedFunction, + 'class': NestedClass}[s]) + + @property + def is_public(self): + """Return True iff this function should be considered public.""" + if self.all is not None: + return self.name in self.all + else: + return not self.name.startswith('_') + + @property + def is_test(self): + """Return True if this function is a test function/method. + + We exclude tests from the imperative mood check, because to phrase + their docstring in the imperative mood, they would have to start with + a highly redundant "Test that ...". + + """ + return self.name.startswith('test') or self.name == 'runTest' + + +class NestedFunction(Function): + """A Python source code nested function.""" + + is_public = False + + +class Method(Function): + """A Python source code method.""" + + @property + def is_magic(self): + """Return True iff this method is a magic method (e.g., `__str__`).""" + return (self.name.startswith('__') and + self.name.endswith('__') and + self.name not in VARIADIC_MAGIC_METHODS) + + @property + def is_init(self): + """Return True iff this method is `__init__`.""" + return self.name == '__init__' + + @property + def is_public(self): + """Return True iff this method should be considered public.""" + # Check if we are a setter/deleter method, and mark as private if so. + for decorator in self.decorators: + # Given 'foo', match 'foo.bar' but not 'foobar' or 'sfoo' + if re(r"^{}\.".format(self.name)).match(decorator.name): + return False + name_is_public = (not self.name.startswith('_') or + self.name in VARIADIC_MAGIC_METHODS or + self.is_magic) + return self.parent.is_public and name_is_public + + +class Class(Definition): + """A Python source code class.""" + + _nest = staticmethod(lambda s: {'def': Method, 'class': NestedClass}[s]) + is_public = Function.is_public + is_class = True + + +class NestedClass(Class): + """A Python source code nested class.""" + + @property + def is_public(self): + """Return True iff this class should be considered public.""" + return (not self.name.startswith('_') and + self.parent.is_class and + self.parent.is_public) + + +class Decorator(Value): + """A decorator for function, method or class.""" + + _fields = 'name arguments'.split() + + +VARIADIC_MAGIC_METHODS = ('__init__', '__call__', '__new__') + + +class AllError(Exception): + """Raised when there is a problem with __all__ when parsing.""" + + def __init__(self, message): + """Initialize the error with a more specific message.""" + Exception.__init__( + self, message + textwrap.dedent(""" + That means pydocstyle cannot decide which definitions are + public. Variable __all__ should be present at most once in + each file, in form + `__all__ = ('a_public_function', 'APublicClass', ...)`. + More info on __all__: http://stackoverflow.com/q/44834/. ') + """)) + + +class TokenStream(object): + # A logical newline is where a new expression or statement begins. When + # there is a physical new line, but not a logical one, for example: + # (x + + # y) + # The token will be tk.NL, not tk.NEWLINE. + LOGICAL_NEWLINES = {tk.NEWLINE, tk.INDENT, tk.DEDENT} + + def __init__(self, filelike): + self._generator = tk.generate_tokens(filelike.readline) + self.current = Token(*next(self._generator, None)) + self.line = self.current.start[0] + self.log = log + self.got_logical_newline = True + + def move(self): + previous = self.current + current = self._next_from_generator() + self.current = None if current is None else Token(*current) + self.line = self.current.start[0] if self.current else self.line + self.got_logical_newline = (previous.kind in self.LOGICAL_NEWLINES) + return previous + + def _next_from_generator(self): + try: + return next(self._generator, None) + except (SyntaxError, tk.TokenError): + self.log.warning('error generating tokens', exc_info=True) + return None + + def __iter__(self): + while True: + if self.current is not None: + yield self.current + else: + return + self.move() + + +class TokenKind(int): + def __repr__(self): + return "tk.{}".format(tk.tok_name[self]) + + +class Token(Value): + _fields = 'kind value start end source'.split() + + def __init__(self, *args): + super(Token, self).__init__(*args) + self.kind = TokenKind(self.kind) + + +class Parser(object): + """A Python source code parser.""" + + def parse(self, filelike, filename): + """Parse the given file-like object and return its Module object.""" + self.log = log + self.source = filelike.readlines() + src = ''.join(self.source) + try: + compile(src, filename, 'exec') + except SyntaxError as error: + six.raise_from(ParseError(), error) + self.stream = TokenStream(StringIO(src)) + self.filename = filename + self.all = None + self.future_imports = set() + self._accumulated_decorators = [] + return self.parse_module() + + # TODO: remove + def __call__(self, *args, **kwargs): + """Call the parse method.""" + return self.parse(*args, **kwargs) + + current = property(lambda self: self.stream.current) + line = property(lambda self: self.stream.line) + + def consume(self, kind): + """Consume one token and verify it is of the expected kind.""" + next_token = self.stream.move() + assert next_token.kind == kind + + def leapfrog(self, kind, value=None): + """Skip tokens in the stream until a certain token kind is reached. + + If `value` is specified, tokens whose values are different will also + be skipped. + """ + while self.current is not None: + if (self.current.kind == kind and + (value is None or self.current.value == value)): + self.consume(kind) + return + self.stream.move() + + def parse_docstring(self): + """Parse a single docstring and return its value.""" + self.log.debug("parsing docstring, token is %r (%s)", + self.current.kind, self.current.value) + while self.current.kind in (tk.COMMENT, tk.NEWLINE, tk.NL): + self.stream.move() + self.log.debug("parsing docstring, token is %r (%s)", + self.current.kind, self.current.value) + if self.current.kind == tk.STRING: + docstring = self.current.value + self.stream.move() + return docstring + return None + + def parse_decorators(self): + """Called after first @ is found. + + Parse decorators into self._accumulated_decorators. + Continue to do so until encountering the 'def' or 'class' start token. + """ + name = [] + arguments = [] + at_arguments = False + + while self.current is not None: + self.log.debug("parsing decorators, current token is %r (%s)", + self.current.kind, self.current.value) + if (self.current.kind == tk.NAME and + self.current.value in ['def', 'class']): + # Done with decorators - found function or class proper + break + elif self.current.kind == tk.OP and self.current.value == '@': + # New decorator found. Store the decorator accumulated so far: + self._accumulated_decorators.append( + Decorator(''.join(name), ''.join(arguments))) + # Now reset to begin accumulating the new decorator: + name = [] + arguments = [] + at_arguments = False + elif self.current.kind == tk.OP and self.current.value == '(': + at_arguments = True + elif self.current.kind == tk.OP and self.current.value == ')': + # Ignore close parenthesis + pass + elif self.current.kind == tk.NEWLINE or self.current.kind == tk.NL: + # Ignore newlines + pass + else: + # Keep accumulating current decorator's name or argument. + if not at_arguments: + name.append(self.current.value) + else: + arguments.append(self.current.value) + self.stream.move() + + # Add decorator accumulated so far + self._accumulated_decorators.append( + Decorator(''.join(name), ''.join(arguments))) + + def parse_definitions(self, class_, all=False): + """Parse multiple definitions and yield them.""" + while self.current is not None: + self.log.debug("parsing definition list, current token is %r (%s)", + self.current.kind, self.current.value) + self.log.debug('got_newline: %s', self.stream.got_logical_newline) + if all and self.current.value == '__all__': + self.parse_all() + elif (self.current.kind == tk.OP and + self.current.value == '@' and + self.stream.got_logical_newline): + self.consume(tk.OP) + self.parse_decorators() + elif self.current.value in ['def', 'class']: + yield self.parse_definition(class_._nest(self.current.value)) + elif self.current.kind == tk.INDENT: + self.consume(tk.INDENT) + for definition in self.parse_definitions(class_): + yield definition + elif self.current.kind == tk.DEDENT: + self.consume(tk.DEDENT) + return + elif self.current.value == 'from': + self.parse_from_import_statement() + else: + self.stream.move() + + def parse_all(self): + """Parse the __all__ definition in a module.""" + assert self.current.value == '__all__' + self.consume(tk.NAME) + if self.current.value != '=': + raise AllError('Could not evaluate contents of __all__. ') + self.consume(tk.OP) + if self.current.value not in '([': + raise AllError('Could not evaluate contents of __all__. ') + self.consume(tk.OP) + + self.all = [] + all_content = "(" + while self.current.kind != tk.OP or self.current.value not in ")]": + if self.current.kind in (tk.NL, tk.COMMENT): + pass + elif (self.current.kind == tk.STRING or + self.current.value == ','): + all_content += self.current.value + else: + raise AllError('Unexpected token kind in __all__: {!r}. ' + .format(self.current.kind)) + self.stream.move() + self.consume(tk.OP) + all_content += ")" + try: + self.all = eval(all_content, {}) + except BaseException as e: + raise AllError('Could not evaluate contents of __all__.' + '\bThe value was {}. The exception was:\n{}' + .format(all_content, e)) + + def parse_module(self): + """Parse a module (and its children) and return a Module object.""" + self.log.debug("parsing module.") + start = self.line + docstring = self.parse_docstring() + children = list(self.parse_definitions(Module, all=True)) + assert self.current is None, self.current + end = self.line + cls = Module + if self.filename.endswith('__init__.py'): + cls = Package + module = cls(self.filename, self.source, start, end, + [], docstring, children, None, self.all, None, '') + for child in module.children: + child.parent = module + module.future_imports = self.future_imports + self.log.debug("finished parsing module.") + return module + + def parse_definition(self, class_): + """Parse a definition and return its value in a `class_` object.""" + start = self.line + self.consume(tk.NAME) + name = self.current.value + self.log.debug("parsing %s '%s'", class_.__name__, name) + self.stream.move() + if self.current.kind == tk.OP and self.current.value == '(': + parenthesis_level = 0 + while True: + if self.current.kind == tk.OP: + if self.current.value == '(': + parenthesis_level += 1 + elif self.current.value == ')': + parenthesis_level -= 1 + if parenthesis_level == 0: + break + self.stream.move() + if self.current.kind != tk.OP or self.current.value != ':': + self.leapfrog(tk.OP, value=":") + else: + self.consume(tk.OP) + if self.current.kind in (tk.NEWLINE, tk.COMMENT): + skipped_error_codes = self.parse_skip_comment() + self.leapfrog(tk.INDENT) + assert self.current.kind != tk.INDENT + docstring = self.parse_docstring() + decorators = self._accumulated_decorators + self.log.debug("current accumulated decorators: %s", decorators) + self._accumulated_decorators = [] + self.log.debug("parsing nested definitions.") + children = list(self.parse_definitions(class_)) + self.log.debug("finished parsing nested definitions for '%s'", + name) + end = self.line - 1 + else: # one-liner definition + skipped_error_codes = '' + docstring = self.parse_docstring() + decorators = [] # TODO + children = [] + end = self.line + self.leapfrog(tk.NEWLINE) + definition = class_(name, self.source, start, end, + decorators, docstring, children, None, + skipped_error_codes) + for child in definition.children: + child.parent = definition + self.log.debug("finished parsing %s '%s'. Next token is %r (%s)", + class_.__name__, name, self.current.kind, + self.current.value) + return definition + + def parse_skip_comment(self): + """Parse a definition comment for noqa skips.""" + skipped_error_codes = '' + if self.current.kind == tk.COMMENT: + if 'noqa: ' in self.current.value: + skipped_error_codes = ''.join( + self.current.value.split('noqa: ')[1:]) + elif self.current.value.startswith('# noqa'): + skipped_error_codes = 'all' + return skipped_error_codes + + def check_current(self, kind=None, value=None): + """Verify the current token is of type `kind` and equals `value`.""" + msg = textwrap.dedent(""" + Unexpected token at line {self.line}: + + In file: {self.filename} + + Got kind {self.current.kind!r} + Got value {self.current.value} + """.format(self=self)) + kind_valid = self.current.kind == kind if kind else True + value_valid = self.current.value == value if value else True + assert kind_valid and value_valid, msg + + def parse_from_import_statement(self): + """Parse a 'from x import y' statement. + + The purpose is to find __future__ statements. + + """ + self.log.debug('parsing from/import statement.') + is_future_import = self._parse_from_import_source() + self._parse_from_import_names(is_future_import) + + def _parse_from_import_source(self): + """Parse the 'from x import' part in a 'from x import y' statement. + + Return true iff `x` is __future__. + """ + assert self.current.value == 'from', self.current.value + self.stream.move() + is_future_import = self.current.value == '__future__' + self.stream.move() + while (self.current is not None and + self.current.kind in (tk.DOT, tk.NAME, tk.OP) and + self.current.value != 'import'): + self.stream.move() + if self.current is None or self.current.value != 'import': + return False + self.check_current(value='import') + assert self.current.value == 'import', self.current.value + self.stream.move() + return is_future_import + + def _parse_from_import_names(self, is_future_import): + """Parse the 'y' part in a 'from x import y' statement.""" + if self.current.value == '(': + self.consume(tk.OP) + expected_end_kinds = (tk.OP, ) + else: + expected_end_kinds = (tk.NEWLINE, tk.ENDMARKER) + while self.current.kind not in expected_end_kinds and not ( + self.current.kind == tk.OP and self.current.value == ';'): + if self.current.kind != tk.NAME: + self.stream.move() + continue + self.log.debug("parsing import, token is %r (%s)", + self.current.kind, self.current.value) + if is_future_import: + self.log.debug('found future import: %s', self.current.value) + self.future_imports.add(self.current.value) + self.consume(tk.NAME) + self.log.debug("parsing import, token is %r (%s)", + self.current.kind, self.current.value) + if self.current.kind == tk.NAME and self.current.value == 'as': + self.consume(tk.NAME) # as + if self.current.kind == tk.NAME: + self.consume(tk.NAME) # new name, irrelevant + if self.current.value == ',': + self.consume(tk.OP) + self.log.debug("parsing import, token is %r (%s)", + self.current.kind, self.current.value) diff --git a/contrib/pydocstyle/utils.py b/contrib/pydocstyle/utils.py new file mode 100644 index 0000000..223a89c --- /dev/null +++ b/contrib/pydocstyle/utils.py @@ -0,0 +1,27 @@ +"""General shared utilities.""" +import logging +from itertools import tee +try: + from itertools import zip_longest +except ImportError: + from itertools import izip_longest as zip_longest + + +# Do not update the version manually - it is managed by `bumpversion`. +__version__ = '2.1.1' +log = logging.getLogger(__name__) + + +def is_blank(string): + """Return True iff the string contains only whitespaces.""" + return not string.strip() + + +def pairwise(iterable, default_value): + """Return pairs of items from `iterable`. + + pairwise([1, 2, 3], default_value=None) -> (1, 2) (2, 3), (3, None) + """ + a, b = tee(iterable) + _ = next(b, default_value) + return zip_longest(a, b, fillvalue=default_value) diff --git a/contrib/pydocstyle/violations.py b/contrib/pydocstyle/violations.py new file mode 100644 index 0000000..18a7e95 --- /dev/null +++ b/contrib/pydocstyle/violations.py @@ -0,0 +1,246 @@ +"""Docstring violation definition.""" +from itertools import dropwhile +from functools import partial +from collections import namedtuple + +from .utils import is_blank + + +__all__ = ('Error', 'ErrorRegistry') + + +ErrorParams = namedtuple('ErrorParams', ['code', 'short_desc', 'context']) + + +class Error(object): + """Error in docstring style.""" + + # Options that define how errors are printed: + explain = False + source = False + + def __init__(self, code, short_desc, context, *parameters): + """Initialize the object. + + `parameters` are specific to the created error. + + """ + self.code = code + self.short_desc = short_desc + self.context = context + self.parameters = parameters + self.definition = None + self.explanation = None + + def set_context(self, definition, explanation): + """Set the source code context for this error.""" + self.definition = definition + self.explanation = explanation + + filename = property(lambda self: self.definition.module.name) + line = property(lambda self: self.definition.start) + + @property + def message(self): + """Return the message to print to the user.""" + ret = '{}: {}'.format(self.code, self.short_desc) + if self.context is not None: + specific_error_msg = self.context.format(*self.parameters) + ret += ' ({})'.format(specific_error_msg) + return ret + + @property + def lines(self): + """Return the source code lines for this error.""" + source = '' + lines = self.definition.source + offset = self.definition.start + lines_stripped = list(reversed(list(dropwhile(is_blank, + reversed(lines))))) + numbers_width = len(str(offset + len(lines_stripped))) + line_format = '{{:{}}}:{{}}'.format(numbers_width) + for n, line in enumerate(lines_stripped): + if line: + line = ' ' + line + source += line_format.format(n + offset, line) + if n > 5: + source += ' ...\n' + break + return source + + def __str__(self): + self.explanation = '\n'.join(l for l in self.explanation.split('\n') + if not is_blank(l)) + template = '{filename}:{line} {definition}:\n {message}' + if self.source and self.explain: + template += '\n\n{explanation}\n\n{lines}\n' + elif self.source and not self.explain: + template += '\n\n{lines}\n' + elif self.explain and not self.source: + template += '\n\n{explanation}\n\n' + return template.format(**dict((name, getattr(self, name)) for name in + ['filename', 'line', 'definition', 'message', + 'explanation', 'lines'])) + + def __repr__(self): + return str(self) + + def __lt__(self, other): + return (self.filename, self.line) < (other.filename, other.line) + + +class ErrorRegistry(object): + """A registry of all error codes, divided to groups.""" + + groups = [] + + class ErrorGroup(object): + """A group of similarly themed errors.""" + + def __init__(self, prefix, name): + """Initialize the object. + + `Prefix` should be the common prefix for errors in this group, + e.g., "D1". + `name` is the name of the group (its subject). + + """ + self.prefix = prefix + self.name = name + self.errors = [] + + def create_error(self, error_code, error_desc, error_context=None): + """Create an error, register it to this group and return it.""" + # TODO: check prefix + + error_params = ErrorParams(error_code, error_desc, error_context) + factory = partial(Error, *error_params) + self.errors.append(error_params) + return factory + + @classmethod + def create_group(cls, prefix, name): + """Create a new error group and return it.""" + group = cls.ErrorGroup(prefix, name) + cls.groups.append(group) + return group + + @classmethod + def get_error_codes(cls): + """Yield all registered codes.""" + for group in cls.groups: + for error in group.errors: + yield error.code + + @classmethod + def to_rst(cls): + """Output the registry as reStructuredText, for documentation.""" + sep_line = '+' + 6 * '-' + '+' + '-' * 71 + '+\n' + blank_line = '|' + 78 * ' ' + '|\n' + table = '' + for group in cls.groups: + table += sep_line + table += blank_line + table += '|' + '**{}**'.format(group.name).center(78) + '|\n' + table += blank_line + for error in group.errors: + table += sep_line + table += ('|' + error.code.center(6) + '| ' + + error.short_desc.ljust(70) + '|\n') + table += sep_line + return table + + +D1xx = ErrorRegistry.create_group('D1', 'Missing Docstrings') +D100 = D1xx.create_error('D100', 'Missing docstring in public module') +D101 = D1xx.create_error('D101', 'Missing docstring in public class') +D102 = D1xx.create_error('D102', 'Missing docstring in public method') +D103 = D1xx.create_error('D103', 'Missing docstring in public function') +D104 = D1xx.create_error('D104', 'Missing docstring in public package') +D105 = D1xx.create_error('D105', 'Missing docstring in magic method') +D106 = D1xx.create_error('D106', 'Missing docstring in public nested class') +D107 = D1xx.create_error('D107', 'Missing docstring in __init__') + +D2xx = ErrorRegistry.create_group('D2', 'Whitespace Issues') +D200 = D2xx.create_error('D200', 'One-line docstring should fit on one line ' + 'with quotes', 'found {0}') +D201 = D2xx.create_error('D201', 'No blank lines allowed before function ' + 'docstring', 'found {0}') +D202 = D2xx.create_error('D202', 'No blank lines allowed after function ' + 'docstring', 'found {0}') +D203 = D2xx.create_error('D203', '1 blank line required before class ' + 'docstring', 'found {0}') +D204 = D2xx.create_error('D204', '1 blank line required after class ' + 'docstring', 'found {0}') +D205 = D2xx.create_error('D205', '1 blank line required between summary line ' + 'and description', 'found {0}') +D206 = D2xx.create_error('D206', 'Docstring should be indented with spaces, ' + 'not tabs') +D207 = D2xx.create_error('D207', 'Docstring is under-indented') +D208 = D2xx.create_error('D208', 'Docstring is over-indented') +D209 = D2xx.create_error('D209', 'Multi-line docstring closing quotes should ' + 'be on a separate line') +D210 = D2xx.create_error('D210', 'No whitespaces allowed surrounding ' + 'docstring text') +D211 = D2xx.create_error('D211', 'No blank lines allowed before class ' + 'docstring', 'found {0}') +D212 = D2xx.create_error('D212', 'Multi-line docstring summary should start ' + 'at the first line') +D213 = D2xx.create_error('D213', 'Multi-line docstring summary should start ' + 'at the second line') +D214 = D2xx.create_error('D214', 'Section is over-indented', '{0!r}') +D215 = D2xx.create_error('D215', 'Section underline is over-indented', + 'in section {0!r}') + +D3xx = ErrorRegistry.create_group('D3', 'Quotes Issues') +D300 = D3xx.create_error('D300', 'Use """triple double quotes"""', + 'found {0}-quotes') +D301 = D3xx.create_error('D301', 'Use r""" if any backslashes in a docstring') +D302 = D3xx.create_error('D302', 'Use u""" for Unicode docstrings') + +D4xx = ErrorRegistry.create_group('D4', 'Docstring Content Issues') +D400 = D4xx.create_error('D400', 'First line should end with a period', + 'not {0!r}') +D401 = D4xx.create_error('D401', 'First line should be in imperative mood', + "'{0}', not '{1}'") +D401b = D4xx.create_error('D401', 'First line should be in imperative mood; ' + 'try rephrasing', "found '{0}'") +D402 = D4xx.create_error('D402', 'First line should not be the function\'s ' + '"signature"') +D403 = D4xx.create_error('D403', 'First word of the first line should be ' + 'properly capitalized', '{0!r}, not {1!r}') +D404 = D4xx.create_error('D404', 'First word of the docstring should not ' + 'be `This`') +D405 = D4xx.create_error('D405', 'Section name should be properly capitalized', + '{0!r}, not {1!r}') +D406 = D4xx.create_error('D406', 'Section name should end with a newline', + '{0!r}, not {1!r}') +D407 = D4xx.create_error('D407', 'Missing dashed underline after section', + '{0!r}') +D408 = D4xx.create_error('D408', 'Section underline should be in the line ' + 'following the section\'s name', + '{0!r}') +D409 = D4xx.create_error('D409', 'Section underline should match the length ' + 'of its name', + 'Expected {0!r} dashes in section {1!r}, got {2!r}') +D410 = D4xx.create_error('D410', 'Missing blank line after section', '{0!r}') +D411 = D4xx.create_error('D411', 'Missing blank line before section', '{0!r}') +D412 = D4xx.create_error('D412', 'No blank lines allowed between a section ' + 'header and its content', '{0!r}') +D413 = D4xx.create_error('D413', 'Missing blank line after last section', + '{0!r}') +D414 = D4xx.create_error('D414', 'Section has no content', '{0!r}') + + +class AttrDict(dict): + def __getattr__(self, item): + return self[item] + +all_errors = set(ErrorRegistry.get_error_codes()) + +conventions = AttrDict({ + 'pep257': all_errors - {'D203', 'D212', 'D213', 'D214', 'D215', 'D404', + 'D405', 'D406', 'D407', 'D408', 'D409', 'D410', + 'D411'}, + 'numpy': all_errors - {'D107', 'D203', 'D212', 'D213', 'D402', 'D413'} +}) diff --git a/contrib/pydocstyle/wordlists.py b/contrib/pydocstyle/wordlists.py new file mode 100644 index 0000000..08f5f3f --- /dev/null +++ b/contrib/pydocstyle/wordlists.py @@ -0,0 +1,39 @@ +"""Wordlists loaded from package data. + +We can treat them as part of the code for the imperative mood check, and +therefore we load them at import time, rather than on-demand. + +""" +import re +import pkgutil +import snowballstemmer + + +#: Regular expression for stripping comments from the wordlists +COMMENT_RE = re.compile(r'\s*#.*') + +#: Stemmer function for stemming words in English +stem = snowballstemmer.stemmer('english').stemWord + + +def load_wordlist(name): + """Iterate over lines of a wordlist data file. + + `name` should be the name of a package data file within the data/ + directory. + + Whitespace and #-prefixed comments are stripped from each line. + + """ + text = pkgutil.get_data('pydocstyle', 'data/' + name).decode('utf8') + for line in text.splitlines(): + line = COMMENT_RE.sub('', line).strip() + if line: + yield line + + +#: A dict mapping stemmed verbs to the imperative form +IMPERATIVE_VERBS = {stem(v): v for v in load_wordlist('imperatives.txt')} + +#: Words that are forbidden to appear as the first word in a docstring +IMPERATIVE_BLACKLIST = set(load_wordlist('imperatives_blacklist.txt')) diff --git a/contrib/pyflakes/__init__.py b/contrib/pyflakes/__init__.py index 1f356cc..bcd8d54 100644 --- a/contrib/pyflakes/__init__.py +++ b/contrib/pyflakes/__init__.py @@ -1 +1 @@ -__version__ = '1.0.0' +__version__ = '1.6.0' diff --git a/contrib/pyflakes/api.py b/contrib/pyflakes/api.py index 3bc2330..49ee38d 100644 --- a/contrib/pyflakes/api.py +++ b/contrib/pyflakes/api.py @@ -5,6 +5,7 @@ import sys import os +import re import _ast from pyflakes import checker, __version__ @@ -13,6 +14,9 @@ __all__ = ['check', 'checkPath', 'checkRecursive', 'iterSourceCode', 'main'] +PYTHON_SHEBANG_REGEX = re.compile(br'^#!.*\bpython[23w]?\b\s*$') + + def check(codeString, filename, reporter=None): """ Check the Python source given by C{codeString} for flakes. @@ -41,6 +45,18 @@ def check(codeString, filename, reporter=None): (lineno, offset, text) = value.lineno, value.offset, value.text + if checker.PYPY: + if text is None: + lines = codeString.splitlines() + if len(lines) >= lineno: + text = lines[lineno - 1] + if sys.version_info >= (3, ) and isinstance(text, bytes): + try: + text = text.decode('ascii') + except UnicodeDecodeError: + text = None + offset -= 1 + # If there's an encoding problem with the file, the text is None. if text is None: # Avoid using msg, since for the only known case, it contains a @@ -96,6 +112,25 @@ def checkPath(filename, reporter=None): return check(codestr, filename, reporter) +def isPythonFile(filename): + """Return True if filename points to a Python file.""" + if filename.endswith('.py'): + return True + + max_bytes = 128 + + try: + with open(filename, 'rb') as f: + text = f.read(max_bytes) + if not text: + return False + except IOError: + return False + + first_line = text.splitlines()[0] + return PYTHON_SHEBANG_REGEX.match(first_line) + + def iterSourceCode(paths): """ Iterate over all Python source files in C{paths}. @@ -108,8 +143,9 @@ def iterSourceCode(paths): if os.path.isdir(path): for dirpath, dirnames, filenames in os.walk(path): for filename in filenames: - if filename.endswith('.py'): - yield os.path.join(dirpath, filename) + full_path = os.path.join(dirpath, filename) + if isPythonFile(full_path): + yield full_path else: yield path @@ -157,7 +193,7 @@ def handler(sig, f): pass -def main(prog=None): +def main(prog=None, args=None): """Entry point for the script "pyflakes".""" import optparse @@ -166,7 +202,7 @@ def main(prog=None): _exitOnSignal('SIGPIPE', 1) parser = optparse.OptionParser(prog=prog, version=__version__) - (__, args) = parser.parse_args() + (__, args) = parser.parse_args(args=args) reporter = modReporter._makeDefaultReporter() if args: warnings = checkRecursive(args, reporter) diff --git a/contrib/pyflakes/checker.py b/contrib/pyflakes/checker.py index e6e1942..75abdc0 100644 --- a/contrib/pyflakes/checker.py +++ b/contrib/pyflakes/checker.py @@ -4,6 +4,7 @@ Implement the central Checker class. Also, it models the Bindings and Scopes. """ +import __future__ import doctest import os import sys @@ -11,6 +12,13 @@ PY2 = sys.version_info < (3, 0) PY32 = sys.version_info < (3, 3) # Python 2.5 to 3.2 PY33 = sys.version_info < (3, 4) # Python 2.5 to 3.3 +PY34 = sys.version_info < (3, 5) # Python 2.5 to 3.4 +try: + sys.pypy_version_info + PYPY = True +except AttributeError: + PYPY = False + builtin_vars = dir(__import__('__builtin__' if PY2 else 'builtins')) try: @@ -48,6 +56,11 @@ def getAlternatives(n): if isinstance(n, ast.Try): return [n.body + n.orelse] + [[hdl] for hdl in n.handlers] +if PY34: + LOOP_TYPES = (ast.While, ast.For) +else: + LOOP_TYPES = (ast.While, ast.For, ast.AsyncFor) + class _FieldsOrder(dict): """Fix order of AST node fields.""" @@ -68,6 +81,17 @@ def __missing__(self, node_class): return fields +def counter(items): + """ + Simplest required implementation of collections.Counter. Required as 2.6 + does not have Counter in collections. + """ + results = {} + for item in items: + results[item] = results.get(item, 0) + 1 + return results + + def iter_child_nodes(node, omit=None, _fields_order=_FieldsOrder()): """ Yield all direct child nodes of *node*, that is, all fields that @@ -84,6 +108,33 @@ def iter_child_nodes(node, omit=None, _fields_order=_FieldsOrder()): yield item +def convert_to_value(item): + if isinstance(item, ast.Str): + return item.s + elif hasattr(ast, 'Bytes') and isinstance(item, ast.Bytes): + return item.s + elif isinstance(item, ast.Tuple): + return tuple(convert_to_value(i) for i in item.elts) + elif isinstance(item, ast.Num): + return item.n + elif isinstance(item, ast.Name): + result = VariableKey(item=item) + constants_lookup = { + 'True': True, + 'False': False, + 'None': None, + } + return constants_lookup.get( + result.name, + result, + ) + elif (not PY33) and isinstance(item, ast.NameConstant): + # None, True, False are nameconstants in python3, but names in 2 + return item.value + else: + return UnhandledKeyType() + + class Binding(object): """ Represents the binding of a value to a name. @@ -92,8 +143,8 @@ class Binding(object): which names have not. See L{Assignment} for a special type of binding that is checked with stricter rules. - @ivar used: pair of (L{Scope}, line-number) indicating the scope and - line number that this binding was last used + @ivar used: pair of (L{Scope}, node) indicating the scope and + the node that this binding was last used. """ def __init__(self, name, source): @@ -120,6 +171,31 @@ class Definition(Binding): """ +class UnhandledKeyType(object): + """ + A dictionary key of a type that we cannot or do not check for duplicates. + """ + + +class VariableKey(object): + """ + A dictionary key which is a variable. + + @ivar item: The variable AST object. + """ + def __init__(self, item): + self.name = item.id + + def __eq__(self, compare): + return ( + compare.__class__ == self.__class__ + and compare.name == self.name + ) + + def __hash__(self): + return hash(self.name) + + class Importation(Definition): """ A binding created by an import statement. @@ -129,17 +205,137 @@ class Importation(Definition): @type fullName: C{str} """ - def __init__(self, name, source): - self.fullName = name + def __init__(self, name, source, full_name=None): + self.fullName = full_name or name self.redefined = [] - name = name.split('.')[0] super(Importation, self).__init__(name, source) def redefines(self, other): - if isinstance(other, Importation): + if isinstance(other, SubmoduleImportation): + # See note in SubmoduleImportation about RedefinedWhileUnused return self.fullName == other.fullName return isinstance(other, Definition) and self.name == other.name + def _has_alias(self): + """Return whether importation needs an as clause.""" + return not self.fullName.split('.')[-1] == self.name + + @property + def source_statement(self): + """Generate a source statement equivalent to the import.""" + if self._has_alias(): + return 'import %s as %s' % (self.fullName, self.name) + else: + return 'import %s' % self.fullName + + def __str__(self): + """Return import full name with alias.""" + if self._has_alias(): + return self.fullName + ' as ' + self.name + else: + return self.fullName + + +class SubmoduleImportation(Importation): + """ + A binding created by a submodule import statement. + + A submodule import is a special case where the root module is implicitly + imported, without an 'as' clause, and the submodule is also imported. + Python does not restrict which attributes of the root module may be used. + + This class is only used when the submodule import is without an 'as' clause. + + pyflakes handles this case by registering the root module name in the scope, + allowing any attribute of the root module to be accessed. + + RedefinedWhileUnused is suppressed in `redefines` unless the submodule + name is also the same, to avoid false positives. + """ + + def __init__(self, name, source): + # A dot should only appear in the name when it is a submodule import + assert '.' in name and (not source or isinstance(source, ast.Import)) + package_name = name.split('.')[0] + super(SubmoduleImportation, self).__init__(package_name, source) + self.fullName = name + + def redefines(self, other): + if isinstance(other, Importation): + return self.fullName == other.fullName + return super(SubmoduleImportation, self).redefines(other) + + def __str__(self): + return self.fullName + + @property + def source_statement(self): + return 'import ' + self.fullName + + +class ImportationFrom(Importation): + + def __init__(self, name, source, module, real_name=None): + self.module = module + self.real_name = real_name or name + + if module.endswith('.'): + full_name = module + self.real_name + else: + full_name = module + '.' + self.real_name + + super(ImportationFrom, self).__init__(name, source, full_name) + + def __str__(self): + """Return import full name with alias.""" + if self.real_name != self.name: + return self.fullName + ' as ' + self.name + else: + return self.fullName + + @property + def source_statement(self): + if self.real_name != self.name: + return 'from %s import %s as %s' % (self.module, + self.real_name, + self.name) + else: + return 'from %s import %s' % (self.module, self.name) + + +class StarImportation(Importation): + """A binding created by a 'from x import *' statement.""" + + def __init__(self, name, source): + super(StarImportation, self).__init__('*', source) + # Each star importation needs a unique name, and + # may not be the module name otherwise it will be deemed imported + self.name = name + '.*' + self.fullName = name + + @property + def source_statement(self): + return 'from ' + self.fullName + ' import *' + + def __str__(self): + # When the module ends with a ., avoid the ambiguous '..*' + if self.fullName.endswith('.'): + return self.source_statement + else: + return self.name + + +class FutureImportation(ImportationFrom): + """ + A binding created by a from `__future__` import statement. + + `__future__` imports are implicitly used. + """ + + def __init__(self, name, source, scope): + super(FutureImportation, self).__init__(name, source, '__future__') + self.used = (scope, source) + class Argument(Binding): """ @@ -237,7 +433,12 @@ class GeneratorScope(Scope): class ModuleScope(Scope): - pass + """Scope for a module.""" + _futures_allowed = True + + +class DoctestScope(ModuleScope): + """Scope for a doctest.""" # Globally defined names which are not attributes of the builtins module, or @@ -249,7 +450,7 @@ def getNodeName(node): # Returns node.id, or node.name, or None if hasattr(node, 'id'): # One of the many nodes with an id return node.id - if hasattr(node, 'name'): # a ExceptHandler node + if hasattr(node, 'name'): # an ExceptHandler node return node.name @@ -289,7 +490,6 @@ def __init__(self, tree, filename='(none)', builtins=None, self.withDoctest = withDoctest self.scopeStack = [ModuleScope()] self.exceptHandlers = [()] - self.futuresAllowed = True self.root = tree self.handleChildren(tree) self.runDeferred(self._deferredFunctions) @@ -331,6 +531,24 @@ def runDeferred(self, deferred): self.offset = offset handler() + def _in_doctest(self): + return (len(self.scopeStack) >= 2 and + isinstance(self.scopeStack[1], DoctestScope)) + + @property + def futuresAllowed(self): + if not all(isinstance(scope, ModuleScope) + for scope in self.scopeStack): + return False + + return self.scope._futures_allowed + + @futuresAllowed.setter + def futuresAllowed(self, value): + assert value is False + if isinstance(self.scope, ModuleScope): + self.scope._futures_allowed = False + @property def scope(self): return self.scopeStack[-1] @@ -344,17 +562,33 @@ def checkDeadScopes(self): which were imported but unused. """ for scope in self.deadScopes: - if isinstance(scope.get('__all__'), ExportBinding): - all_names = set(scope['__all__'].names) + # imports in classes are public members + if isinstance(scope, ClassScope): + continue + + all_binding = scope.get('__all__') + if all_binding and not isinstance(all_binding, ExportBinding): + all_binding = None + + if all_binding: + all_names = set(all_binding.names) + undefined = all_names.difference(scope) + else: + all_names = undefined = [] + + if undefined: if not scope.importStarred and \ os.path.basename(self.filename) != '__init__.py': # Look for possible mistakes in the export list - undefined = all_names.difference(scope) for name in undefined: self.report(messages.UndefinedExport, scope['__all__'].source, name) - else: - all_names = [] + + # mark all import '*' as used by the undefined in __all__ + if scope.importStarred: + for binding in scope.values(): + if isinstance(binding, StarImportation): + binding.used = all_binding # Look for imported names that aren't used. for value in scope.values(): @@ -362,7 +596,7 @@ def checkDeadScopes(self): used = value.used or value.name in all_names if not used: messg = messages.UnusedImport - self.report(messg, value.source, value.name) + self.report(messg, value.source, str(value)) for node in value.redefined: if isinstance(self.getParent(node), ast.For): messg = messages.ImportShadowedByLoopVar @@ -466,23 +700,17 @@ def handleNodeLoad(self, node): name = getNodeName(node) if not name: return - # try local scope - try: - self.scope[name].used = (self.scope, node) - except KeyError: - pass - else: - return - scopes = [scope for scope in self.scopeStack[:-1] - if isinstance(scope, (FunctionScope, ModuleScope, GeneratorScope))] - if isinstance(self.scope, GeneratorScope) and scopes[-1] != self.scopeStack[-2]: - scopes.append(self.scopeStack[-2]) + in_generators = None + importStarred = None # try enclosing function scopes and global scope - importStarred = self.scope.importStarred - for scope in reversed(scopes): - importStarred = importStarred or scope.importStarred + for scope in self.scopeStack[-1::-1]: + # only generators used in a class scope can access the names + # of the class. this is skipped during the first iteration + if in_generators is False and isinstance(scope, ClassScope): + continue + try: scope[name].used = (self.scope, node) except KeyError: @@ -490,9 +718,30 @@ def handleNodeLoad(self, node): else: return + importStarred = importStarred or scope.importStarred + + if in_generators is not False: + in_generators = isinstance(scope, GeneratorScope) + # look in the built-ins - if importStarred or name in self.builtIns: + if name in self.builtIns: + return + + if importStarred: + from_list = [] + + for scope in self.scopeStack[-1::-1]: + for binding in scope.values(): + if isinstance(binding, StarImportation): + # mark '*' imports as used for each scope + binding.used = (self.scope, node) + from_list.append(binding.fullName) + + # report * usage, with a list of possible sources + from_list = ', '.join(sorted(from_list)) + self.report(messages.ImportStarUsage, node, name, from_list) return + if name == '__path__' and os.path.basename(self.filename) == '__init__.py': # the special name __path__ is valid only in packages return @@ -550,7 +799,7 @@ def on_conditional_branch(): return if on_conditional_branch(): - # We can not predict if this conditional branch is going to + # We cannot predict if this conditional branch is going to # be executed. return @@ -586,8 +835,13 @@ def getDocstring(self, node): node = node.value if not isinstance(node, ast.Str): return (None, None) - # Computed incorrectly if the docstring has backslash - doctest_lineno = node.lineno - node.s.count('\n') - 1 + + if PYPY: + doctest_lineno = node.lineno - 1 + else: + # Computed incorrectly if the docstring has backslash + doctest_lineno = node.lineno - node.s.count('\n') - 1 + return (node.s, doctest_lineno) def handleNode(self, node, parent): @@ -616,7 +870,19 @@ def handleNode(self, node, parent): def handleDoctests(self, node): try: - (docstring, node_lineno) = self.getDocstring(node.body[0]) + if hasattr(node, 'docstring'): + docstring = node.docstring + + # This is just a reasonable guess. In Python 3.7, docstrings no + # longer have line numbers associated with them. This will be + # incorrect if there are empty lines between the beginning + # of the function and the docstring. + node_lineno = node.lineno + if hasattr(node, 'args'): + node_lineno = max([node_lineno] + + [arg.lineno for arg in node.args.args]) + else: + (docstring, node_lineno) = self.getDocstring(node.body[0]) examples = docstring and self._getDoctestExamples(docstring) except (ValueError, IndexError): # e.g. line 6 of the docstring for has inconsistent @@ -624,8 +890,12 @@ def handleDoctests(self, node): return if not examples: return + + # Place doctest in module scope + saved_stack = self.scopeStack + self.scopeStack = [self.scopeStack[0]] node_offset = self.offset or (0, 0) - self.pushScope() + self.pushScope(DoctestScope) underscore_in_builtins = '_' in self.builtIns if not underscore_in_builtins: self.builtIns.add('_') @@ -634,6 +904,8 @@ def handleDoctests(self, node): tree = compile(example.source, "", "exec", ast.PyCF_ONLY_AST) except SyntaxError: e = sys.exc_info()[1] + if PYPY: + e.offset += 1 position = (node_lineno + example.lineno + e.lineno, example.indent + 4 + (e.offset or 0)) self.report(messages.DoctestSyntaxError, node, position) @@ -645,20 +917,21 @@ def handleDoctests(self, node): if not underscore_in_builtins: self.builtIns.remove('_') self.popScope() + self.scopeStack = saved_stack def ignore(self, node): pass # "stmt" type nodes DELETE = PRINT = FOR = ASYNCFOR = WHILE = IF = WITH = WITHITEM = \ - ASYNCWITH = ASYNCWITHITEM = RAISE = TRYFINALLY = ASSERT = EXEC = \ + ASYNCWITH = ASYNCWITHITEM = RAISE = TRYFINALLY = EXEC = \ EXPR = ASSIGN = handleChildren - CONTINUE = BREAK = PASS = ignore + PASS = ignore # "expr" type nodes - BOOLOP = BINOP = UNARYOP = IFEXP = DICT = SET = \ - COMPARE = CALL = REPR = ATTRIBUTE = SUBSCRIPT = LIST = TUPLE = \ + BOOLOP = BINOP = UNARYOP = IFEXP = SET = \ + COMPARE = CALL = REPR = ATTRIBUTE = SUBSCRIPT = \ STARRED = NAMECONSTANT = handleChildren NUM = STR = BYTES = ELLIPSIS = ignore @@ -672,17 +945,58 @@ def ignore(self, node): # same for operators AND = OR = ADD = SUB = MULT = DIV = MOD = POW = LSHIFT = RSHIFT = \ BITOR = BITXOR = BITAND = FLOORDIV = INVERT = NOT = UADD = USUB = \ - EQ = NOTEQ = LT = LTE = GT = GTE = IS = ISNOT = IN = NOTIN = ignore + EQ = NOTEQ = LT = LTE = GT = GTE = IS = ISNOT = IN = NOTIN = \ + MATMULT = ignore # additional node types - COMPREHENSION = KEYWORD = handleChildren + COMPREHENSION = KEYWORD = FORMATTEDVALUE = JOINEDSTR = handleChildren + + def DICT(self, node): + # Complain if there are duplicate keys with different values + # If they have the same value it's not going to cause potentially + # unexpected behaviour so we'll not complain. + keys = [ + convert_to_value(key) for key in node.keys + ] + + key_counts = counter(keys) + duplicate_keys = [ + key for key, count in key_counts.items() + if count > 1 + ] + + for key in duplicate_keys: + key_indices = [i for i, i_key in enumerate(keys) if i_key == key] + + values = counter( + convert_to_value(node.values[index]) + for index in key_indices + ) + if any(count == 1 for value, count in values.items()): + for key_index in key_indices: + key_node = node.keys[key_index] + if isinstance(key, VariableKey): + self.report(messages.MultiValueRepeatedKeyVariable, + key_node, + key.name) + else: + self.report( + messages.MultiValueRepeatedKeyLiteral, + key_node, + key, + ) + self.handleChildren(node) + + def ASSERT(self, node): + if isinstance(node.test, ast.Tuple) and node.test.elts != []: + self.report(messages.AssertTuple, node) + self.handleChildren(node) def GLOBAL(self, node): """ Keep track of globals declarations. """ - # In doctests, the global scope is an anonymous function at index 1. - global_scope_index = 1 if self.withDoctest else 0 + global_scope_index = 1 if self._in_doctest() else 0 global_scope = self.scopeStack[global_scope_index] # Ignore 'global' statement in global scope. @@ -693,10 +1007,12 @@ def GLOBAL(self, node): node_value = Assignment(node_name, node) # Remove UndefinedName messages already reported for this name. + # TODO: if the global is not used in this scope, it does not + # become a globally defined name. See test_unused_global. self.messages = [ m for m in self.messages if not - isinstance(m, messages.UndefinedName) and not - m.message_args[0] == node_name] + isinstance(m, messages.UndefinedName) or + m.message_args[0] != node_name] # Bind name to global scope if it doesn't exist already. global_scope.setdefault(node_name, node_value) @@ -737,8 +1053,33 @@ def NAME(self, node): # arguments, but these aren't dispatched through here raise RuntimeError("Got impossible expression context: %r" % (node.ctx,)) + def CONTINUE(self, node): + # Walk the tree up until we see a loop (OK), a function or class + # definition (not OK), for 'continue', a finally block (not OK), or + # the top module scope (not OK) + n = node + while hasattr(n, 'parent'): + n, n_child = n.parent, n + if isinstance(n, LOOP_TYPES): + # Doesn't apply unless it's in the loop itself + if n_child not in n.orelse: + return + if isinstance(n, (ast.FunctionDef, ast.ClassDef)): + break + # Handle Try/TryFinally difference in Python < and >= 3.3 + if hasattr(n, 'finalbody') and isinstance(node, ast.Continue): + if n_child in n.finalbody: + self.report(messages.ContinueInFinally, node) + return + if isinstance(node, ast.Continue): + self.report(messages.ContinueOutsideLoop, node) + else: # ast.Break + self.report(messages.BreakOutsideLoop, node) + + BREAK = CONTINUE + def RETURN(self, node): - if isinstance(self.scope, ClassScope): + if isinstance(self.scope, (ClassScope, ModuleScope)): self.report(messages.ReturnOutsideFunction, node) return @@ -751,6 +1092,10 @@ def RETURN(self, node): self.handleNode(node.value, node) def YIELD(self, node): + if isinstance(self.scope, (ClassScope, ModuleScope)): + self.report(messages.YieldOutsideFunction, node) + return + self.scope.isGenerator = True self.handleNode(node.value, node) @@ -761,7 +1106,11 @@ def FUNCTIONDEF(self, node): self.handleNode(deco, node) self.LAMBDA(node) self.addBinding(node, FunctionDefinition(node.name, node)) - if self.withDoctest: + # doctest does not process doctest within a doctest, + # or in nested functions. + if (self.withDoctest and + not self._in_doctest() and + not isinstance(self.scope, FunctionScope)): self.deferFunction(lambda: self.handleDoctests(node)) ASYNCFUNCTIONDEF = FUNCTIONDEF @@ -861,7 +1210,11 @@ def CLASSDEF(self, node): for keywordNode in node.keywords: self.handleNode(keywordNode, node) self.pushScope(ClassScope) - if self.withDoctest: + # doctest does not process doctest within a doctest + # classes within classes are processed. + if (self.withDoctest and + not self._in_doctest() and + not isinstance(self.scope, FunctionScope)): self.deferFunction(lambda: self.handleDoctests(node)) for stmt in node.body: self.handleNode(stmt, node) @@ -873,10 +1226,38 @@ def AUGASSIGN(self, node): self.handleNode(node.value, node) self.handleNode(node.target, node) + def TUPLE(self, node): + if not PY2 and isinstance(node.ctx, ast.Store): + # Python 3 advanced tuple unpacking: a, *b, c = d. + # Only one starred expression is allowed, and no more than 1<<8 + # assignments are allowed before a stared expression. There is + # also a limit of 1<<24 expressions after the starred expression, + # which is impossible to test due to memory restrictions, but we + # add it here anyway + has_starred = False + star_loc = -1 + for i, n in enumerate(node.elts): + if isinstance(n, ast.Starred): + if has_starred: + self.report(messages.TwoStarredExpressions, node) + # The SyntaxError doesn't distinguish two from more + # than two. + break + has_starred = True + star_loc = i + if star_loc >= 1 << 8 or len(node.elts) - star_loc - 1 >= 1 << 24: + self.report(messages.TooManyExpressionsInStarredAssignment, node) + self.handleChildren(node) + + LIST = TUPLE + def IMPORT(self, node): for alias in node.names: - name = alias.asname or alias.name - importation = Importation(name, node) + if '.' in alias.name and not alias.asname: + importation = SubmoduleImportation(alias.name, node) + else: + name = alias.asname or alias.name + importation = Importation(name, node, alias.name) self.addBinding(node, importation) def IMPORTFROM(self, node): @@ -887,26 +1268,42 @@ def IMPORTFROM(self, node): else: self.futuresAllowed = False + module = ('.' * node.level) + (node.module or '') + for alias in node.names: - if alias.name == '*': - self.scope.importStarred = True - self.report(messages.ImportStarUsed, node, node.module) - continue name = alias.asname or alias.name - importation = Importation(name, node) if node.module == '__future__': - importation.used = (self.scope, node) + importation = FutureImportation(name, node, self.scope) + if alias.name not in __future__.all_feature_names: + self.report(messages.FutureFeatureNotDefined, + node, alias.name) + elif alias.name == '*': + # Only Python 2, local import * is a SyntaxWarning + if not PY2 and not isinstance(self.scope, ModuleScope): + self.report(messages.ImportStarNotPermitted, + node, module) + continue + + self.scope.importStarred = True + self.report(messages.ImportStarUsed, node, module) + importation = StarImportation(module, node) + else: + importation = ImportationFrom(name, node, + module, alias.name) self.addBinding(node, importation) def TRY(self, node): handler_names = [] # List the exception handlers - for handler in node.handlers: + for i, handler in enumerate(node.handlers): if isinstance(handler.type, ast.Tuple): for exc_type in handler.type.elts: handler_names.append(getNodeName(exc_type)) elif handler.type: handler_names.append(getNodeName(handler.type)) + + if handler.type is None and i < len(node.handlers) - 1: + self.report(messages.DefaultExceptNotLast, handler) # Memorize the except handlers and process the body self.exceptHandlers.append(handler_names) for child in node.body: @@ -918,8 +1315,46 @@ def TRY(self, node): TRYEXCEPT = TRY def EXCEPTHANDLER(self, node): - # 3.x: in addition to handling children, we must handle the name of - # the exception, which is not a Name node, but a simple string. - if isinstance(node.name, str): - self.handleNodeStore(node) + if PY2 or node.name is None: + self.handleChildren(node) + return + + # 3.x: the name of the exception, which is not a Name node, but + # a simple string, creates a local that is only bound within the scope + # of the except: block. + + for scope in self.scopeStack[::-1]: + if node.name in scope: + is_name_previously_defined = True + break + else: + is_name_previously_defined = False + + self.handleNodeStore(node) self.handleChildren(node) + if not is_name_previously_defined: + # See discussion on https://github.com/PyCQA/pyflakes/pull/59 + + # We're removing the local name since it's being unbound + # after leaving the except: block and it's always unbound + # if the except: block is never entered. This will cause an + # "undefined name" error raised if the checked code tries to + # use the name afterwards. + # + # Unless it's been removed already. Then do nothing. + + try: + del self.scope[node.name] + except KeyError: + pass + + def ANNASSIGN(self, node): + if node.value: + # Only bind the *targets* if the assignment has a value. + # Otherwise it's not really ast.Store and shouldn't silence + # UndefinedLocal warnings. + self.handleNode(node.target, node) + self.handleNode(node.annotation, node) + if node.value: + # If the assignment has value, handle the *value* now. + self.handleNode(node.value, node) diff --git a/contrib/pyflakes/messages.py b/contrib/pyflakes/messages.py index 8899b7b..9e9406c 100644 --- a/contrib/pyflakes/messages.py +++ b/contrib/pyflakes/messages.py @@ -49,6 +49,14 @@ def __init__(self, filename, loc, name, orig_loc): self.message_args = (name, orig_loc.lineno) +class ImportStarNotPermitted(Message): + message = "'from %s import *' only allowed at module level" + + def __init__(self, filename, loc, modname): + Message.__init__(self, filename, loc) + self.message_args = (modname,) + + class ImportStarUsed(Message): message = "'from %s import *' used; unable to detect undefined names" @@ -57,6 +65,14 @@ def __init__(self, filename, loc, modname): self.message_args = (modname,) +class ImportStarUsage(Message): + message = "%r may be undefined, or defined from star imports: %s" + + def __init__(self, filename, loc, name, from_list): + Message.__init__(self, filename, loc) + self.message_args = (name, from_list) + + class UndefinedName(Message): message = 'undefined name %r' @@ -100,17 +116,42 @@ def __init__(self, filename, loc, name): self.message_args = (name,) +class MultiValueRepeatedKeyLiteral(Message): + message = 'dictionary key %r repeated with different values' + + def __init__(self, filename, loc, key): + Message.__init__(self, filename, loc) + self.message_args = (key,) + + +class MultiValueRepeatedKeyVariable(Message): + message = 'dictionary key variable %s repeated with different values' + + def __init__(self, filename, loc, key): + Message.__init__(self, filename, loc) + self.message_args = (key,) + + class LateFutureImport(Message): - message = 'future import(s) %r after other statements' + message = 'from __future__ imports must occur at the beginning of the file' def __init__(self, filename, loc, names): Message.__init__(self, filename, loc) - self.message_args = (names,) + self.message_args = () + + +class FutureFeatureNotDefined(Message): + """An undefined __future__ feature name was imported.""" + message = 'future feature %s is not defined' + + def __init__(self, filename, loc, name): + Message.__init__(self, filename, loc) + self.message_args = (name,) class UnusedVariable(Message): """ - Indicates that a variable has been explicity assigned to but not actually + Indicates that a variable has been explicitly assigned to but not actually used. """ message = 'local variable %r is assigned to but never used' @@ -132,3 +173,61 @@ class ReturnOutsideFunction(Message): Indicates a return statement outside of a function/method. """ message = '\'return\' outside function' + + +class YieldOutsideFunction(Message): + """ + Indicates a yield or yield from statement outside of a function/method. + """ + message = '\'yield\' outside function' + + +# For whatever reason, Python gives different error messages for these two. We +# match the Python error message exactly. +class ContinueOutsideLoop(Message): + """ + Indicates a continue statement outside of a while or for loop. + """ + message = '\'continue\' not properly in loop' + + +class BreakOutsideLoop(Message): + """ + Indicates a break statement outside of a while or for loop. + """ + message = '\'break\' outside loop' + + +class ContinueInFinally(Message): + """ + Indicates a continue statement in a finally block in a while or for loop. + """ + message = '\'continue\' not supported inside \'finally\' clause' + + +class DefaultExceptNotLast(Message): + """ + Indicates an except: block as not the last exception handler. + """ + message = 'default \'except:\' must be last' + + +class TwoStarredExpressions(Message): + """ + Two or more starred expressions in an assignment (a, *b, *c = d). + """ + message = 'two starred expressions in assignment' + + +class TooManyExpressionsInStarredAssignment(Message): + """ + Too many expressions in an assignment with star-unpacking + """ + message = 'too many expressions in star-unpacking assignment' + + +class AssertTuple(Message): + """ + Assertion test is a tuple, which are always True. + """ + message = 'assertion is always true, perhaps remove parentheses?' diff --git a/contrib/pyflakes/reporter.py b/contrib/pyflakes/reporter.py index ae645bd..8b56e74 100644 --- a/contrib/pyflakes/reporter.py +++ b/contrib/pyflakes/reporter.py @@ -38,7 +38,7 @@ def unexpectedError(self, filename, msg): def syntaxError(self, filename, msg, lineno, offset, text): """ - There was a syntax errror in C{filename}. + There was a syntax error in C{filename}. @param filename: The path to the file with the syntax error. @ptype filename: C{unicode} diff --git a/contrib/pyflakes/test/harness.py b/contrib/pyflakes/test/harness.py index a781237..009923f 100644 --- a/contrib/pyflakes/test/harness.py +++ b/contrib/pyflakes/test/harness.py @@ -36,8 +36,37 @@ def flakes(self, input, *expectedOutputs, **kw): %s''' % (input, expectedOutputs, '\n'.join([str(o) for o in w.messages]))) return w - if sys.version_info < (2, 7): + if not hasattr(unittest.TestCase, 'assertIs'): def assertIs(self, expr1, expr2, msg=None): if expr1 is not expr2: self.fail(msg or '%r is not %r' % (expr1, expr2)) + + if not hasattr(unittest.TestCase, 'assertIsInstance'): + + def assertIsInstance(self, obj, cls, msg=None): + """Same as self.assertTrue(isinstance(obj, cls)).""" + if not isinstance(obj, cls): + self.fail(msg or '%r is not an instance of %r' % (obj, cls)) + + if not hasattr(unittest.TestCase, 'assertNotIsInstance'): + + def assertNotIsInstance(self, obj, cls, msg=None): + """Same as self.assertFalse(isinstance(obj, cls)).""" + if isinstance(obj, cls): + self.fail(msg or '%r is an instance of %r' % (obj, cls)) + + if not hasattr(unittest.TestCase, 'assertIn'): + + def assertIn(self, member, container, msg=None): + """Just like self.assertTrue(a in b).""" + if member not in container: + self.fail(msg or '%r not found in %r' % (member, container)) + + if not hasattr(unittest.TestCase, 'assertNotIn'): + + def assertNotIn(self, member, container, msg=None): + """Just like self.assertTrue(a not in b).""" + if member in container: + self.fail(msg or + '%r unexpectedly found in %r' % (member, container)) diff --git a/contrib/pyflakes/test/test_api.py b/contrib/pyflakes/test/test_api.py index 34a59bc..3f54ca4 100644 --- a/contrib/pyflakes/test/test_api.py +++ b/contrib/pyflakes/test/test_api.py @@ -8,9 +8,11 @@ import subprocess import tempfile +from pyflakes.checker import PY2 from pyflakes.messages import UnusedImport from pyflakes.reporter import Reporter from pyflakes.api import ( + main, checkPath, checkRecursive, iterSourceCode, @@ -23,6 +25,20 @@ from io import StringIO unichr = chr +try: + sys.pypy_version_info + PYPY = True +except AttributeError: + PYPY = False + +try: + WindowsError + WIN = True +except NameError: + WIN = False + +ERROR_HAS_COL_NUM = ERROR_HAS_LAST_LINE = sys.version_info >= (3, 2) or PYPY + def withStderrTo(stderr, f, *args, **kwargs): """ @@ -44,6 +60,57 @@ def __init__(self, lineno, col_offset=0): self.col_offset = col_offset +class SysStreamCapturing(object): + + """ + Context manager capturing sys.stdin, sys.stdout and sys.stderr. + + The file handles are replaced with a StringIO object. + On environments that support it, the StringIO object uses newlines + set to os.linesep. Otherwise newlines are converted from \\n to + os.linesep during __exit__. + """ + + def _create_StringIO(self, buffer=None): + # Python 3 has a newline argument + try: + return StringIO(buffer, newline=os.linesep) + except TypeError: + self._newline = True + # Python 2 creates an input only stream when buffer is not None + if buffer is None: + return StringIO() + else: + return StringIO(buffer) + + def __init__(self, stdin): + self._newline = False + self._stdin = self._create_StringIO(stdin or '') + + def __enter__(self): + self._orig_stdin = sys.stdin + self._orig_stdout = sys.stdout + self._orig_stderr = sys.stderr + + sys.stdin = self._stdin + sys.stdout = self._stdout_stringio = self._create_StringIO() + sys.stderr = self._stderr_stringio = self._create_StringIO() + + return self + + def __exit__(self, *args): + self.output = self._stdout_stringio.getvalue() + self.error = self._stderr_stringio.getvalue() + + if self._newline and os.linesep != '\n': + self.output = self.output.replace('\n', os.linesep) + self.error = self.error.replace('\n', os.linesep) + + sys.stdin = self._orig_stdin + sys.stdout = self._orig_stdout + sys.stderr = self._orig_stderr + + class LoggingReporter(object): """ Implementation of Reporter that just appends any error to a list. @@ -120,6 +187,36 @@ def test_recurses(self): sorted(iterSourceCode([self.tempdir])), sorted([apath, bpath, cpath])) + def test_shebang(self): + """ + Find Python files that don't end with `.py`, but contain a Python + shebang. + """ + python = os.path.join(self.tempdir, 'a') + with open(python, 'w') as fd: + fd.write('#!/usr/bin/env python\n') + + self.makeEmptyFile('b') + + with open(os.path.join(self.tempdir, 'c'), 'w') as fd: + fd.write('hello\nworld\n') + + python2 = os.path.join(self.tempdir, 'd') + with open(python2, 'w') as fd: + fd.write('#!/usr/bin/env python2\n') + + python3 = os.path.join(self.tempdir, 'e') + with open(python3, 'w') as fd: + fd.write('#!/usr/bin/env python3\n') + + pythonw = os.path.join(self.tempdir, 'f') + with open(pythonw, 'w') as fd: + fd.write('#!/usr/bin/env pythonw\n') + + self.assertEqual( + sorted(iterSourceCode([self.tempdir])), + sorted([python, python2, python3, pythonw])) + def test_multipleDirectories(self): """ L{iterSourceCode} can be given multiple directories. It will recurse @@ -312,18 +409,25 @@ def evaluate(source): evaluate(source) except SyntaxError: e = sys.exc_info()[1] - self.assertTrue(e.text.count('\n') > 1) + if not PYPY: + self.assertTrue(e.text.count('\n') > 1) else: self.fail() sourcePath = self.makeTempFile(source) + + if PYPY: + message = 'EOF while scanning triple-quoted string literal' + else: + message = 'invalid syntax' + self.assertHasErrors( sourcePath, ["""\ -%s:8:11: invalid syntax +%s:8:11: %s '''quux''' ^ -""" % (sourcePath,)]) +""" % (sourcePath, message)]) def test_eofSyntaxError(self): """ @@ -331,13 +435,22 @@ def test_eofSyntaxError(self): syntax error reflects the cause for the syntax error. """ sourcePath = self.makeTempFile("def foo(") - self.assertHasErrors( - sourcePath, - ["""\ + if PYPY: + result = """\ +%s:1:7: parenthesis is never closed +def foo( + ^ +""" % (sourcePath,) + else: + result = """\ %s:1:9: unexpected EOF while parsing def foo( ^ -""" % (sourcePath,)]) +""" % (sourcePath,) + + self.assertHasErrors( + sourcePath, + [result]) def test_eofSyntaxErrorWithTab(self): """ @@ -345,13 +458,16 @@ def test_eofSyntaxErrorWithTab(self): syntax error reflects the cause for the syntax error. """ sourcePath = self.makeTempFile("if True:\n\tfoo =") + column = 5 if PYPY else 7 + last_line = '\t ^' if PYPY else '\t ^' + self.assertHasErrors( sourcePath, ["""\ -%s:2:7: invalid syntax +%s:2:%s: invalid syntax \tfoo = -\t ^ -""" % (sourcePath,)]) +%s +""" % (sourcePath, column, last_line)]) def test_nonDefaultFollowsDefaultSyntaxError(self): """ @@ -364,8 +480,8 @@ def foo(bar=baz, bax): pass """ sourcePath = self.makeTempFile(source) - last_line = ' ^\n' if sys.version_info >= (3, 2) else '' - column = '8:' if sys.version_info >= (3, 2) else '' + last_line = ' ^\n' if ERROR_HAS_LAST_LINE else '' + column = '8:' if ERROR_HAS_COL_NUM else '' self.assertHasErrors( sourcePath, ["""\ @@ -383,8 +499,8 @@ def test_nonKeywordAfterKeywordSyntaxError(self): foo(bar=baz, bax) """ sourcePath = self.makeTempFile(source) - last_line = ' ^\n' if sys.version_info >= (3, 2) else '' - column = '13:' if sys.version_info >= (3, 2) else '' + last_line = ' ^\n' if ERROR_HAS_LAST_LINE else '' + column = '13:' if ERROR_HAS_COL_NUM or PYPY else '' if sys.version_info >= (3, 5): message = 'positional argument follows keyword argument' @@ -407,8 +523,15 @@ def test_invalidEscape(self): sourcePath = self.makeTempFile(r"foo = '\xyz'") if ver < (3,): decoding_error = "%s: problem decoding source\n" % (sourcePath,) + elif PYPY: + # pypy3 only + decoding_error = """\ +%s:1:6: %s: ('unicodeescape', b'\\\\xyz', 0, 2, 'truncated \\\\xXX escape') +foo = '\\xyz' + ^ +""" % (sourcePath, 'UnicodeDecodeError') else: - last_line = ' ^\n' if ver >= (3, 2) else '' + last_line = ' ^\n' if ERROR_HAS_LAST_LINE else '' # Column has been "fixed" since 3.2.4 and 3.3.1 col = 1 if ver >= (3, 3, 1) or ((3, 2, 4) <= ver < (3, 3)) else 2 decoding_error = """\ @@ -425,6 +548,9 @@ def test_permissionDenied(self): If the source file is not readable, this is reported on standard error. """ + if os.getuid() == 0: + self.skipTest('root user can access all files regardless of ' + 'permissions') sourcePath = self.makeTempFile('') os.chmod(sourcePath, 0) count, errors = self.getErrors(sourcePath) @@ -474,8 +600,21 @@ def test_misencodedFileUTF8(self): x = "%s" """ % SNOWMAN).encode('utf-8') sourcePath = self.makeTempFile(source) + + if PYPY and sys.version_info < (3, ): + message = ('\'ascii\' codec can\'t decode byte 0xe2 ' + 'in position 21: ordinal not in range(128)') + result = """\ +%s:0:0: %s +x = "\xe2\x98\x83" + ^\n""" % (sourcePath, message) + + else: + message = 'problem decoding source' + result = "%s: problem decoding source\n" % (sourcePath,) + self.assertHasErrors( - sourcePath, ["%s: problem decoding source\n" % (sourcePath,)]) + sourcePath, [result]) def test_misencodedFileUTF16(self): """ @@ -541,8 +680,8 @@ def runPyflakes(self, paths, stdin=None): """ Launch a subprocess running C{pyflakes}. - @param args: Command-line arguments to pass to pyflakes. - @param kwargs: Options passed on to C{subprocess.Popen}. + @param paths: Command-line arguments to pass to pyflakes. + @param stdin: Text to use as stdin. @return: C{(returncode, stdout, stderr)} of the completed pyflakes process. """ @@ -553,7 +692,7 @@ def runPyflakes(self, paths, stdin=None): if stdin: p = subprocess.Popen(command, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = p.communicate(stdin) + (stdout, stderr) = p.communicate(stdin.encode('ascii')) else: p = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -562,6 +701,9 @@ def runPyflakes(self, paths, stdin=None): if sys.version_info >= (3,): stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') + # Workaround https://bitbucket.org/pypy/pypy/issues/2350 + if PYPY and PY2 and WIN: + stderr = stderr.replace('\r\r\n', '\r\n') return (stdout, stderr, rv) def test_goodFile(self): @@ -586,7 +728,7 @@ def test_fileWithFlakes(self): expected = UnusedImport(self.tempfilepath, Node(1), 'contraband') self.assertEqual(d, ("%s%s" % (expected, os.linesep), '', 1)) - def test_errors(self): + def test_errors_io(self): """ When pyflakes finds errors with the files it's given, (if they don't exist, say), then the return code is non-zero and the errors are @@ -597,10 +739,39 @@ def test_errors(self): os.linesep) self.assertEqual(d, ('', error_msg, 1)) + def test_errors_syntax(self): + """ + When pyflakes finds errors with the files it's given, (if they don't + exist, say), then the return code is non-zero and the errors are + printed to stderr. + """ + fd = open(self.tempfilepath, 'wb') + fd.write("import".encode('ascii')) + fd.close() + d = self.runPyflakes([self.tempfilepath]) + error_msg = '{0}:1:{2}: invalid syntax{1}import{1} {3}^{1}'.format( + self.tempfilepath, os.linesep, 5 if PYPY else 7, '' if PYPY else ' ') + self.assertEqual(d, ('', error_msg, True)) + def test_readFromStdin(self): """ If no arguments are passed to C{pyflakes} then it reads from stdin. """ - d = self.runPyflakes([], stdin='import contraband'.encode('ascii')) + d = self.runPyflakes([], stdin='import contraband') expected = UnusedImport('', Node(1), 'contraband') self.assertEqual(d, ("%s%s" % (expected, os.linesep), '', 1)) + + +class TestMain(IntegrationTests): + """ + Tests of the pyflakes main function. + """ + + def runPyflakes(self, paths, stdin=None): + try: + with SysStreamCapturing(stdin) as capture: + main(args=paths) + except SystemExit as e: + return (capture.output, capture.error, e.code) + else: + raise RuntimeError('SystemExit not raised') diff --git a/contrib/pyflakes/test/test_dict.py b/contrib/pyflakes/test/test_dict.py new file mode 100644 index 0000000..628ec0c --- /dev/null +++ b/contrib/pyflakes/test/test_dict.py @@ -0,0 +1,217 @@ +""" +Tests for dict duplicate keys Pyflakes behavior. +""" + +from sys import version_info + +from pyflakes import messages as m +from pyflakes.test.harness import TestCase, skipIf + + +class Test(TestCase): + + def test_duplicate_keys(self): + self.flakes( + "{'yes': 1, 'yes': 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + @skipIf(version_info < (3,), + "bytes and strings with same 'value' are not equal in python3") + @skipIf(version_info[0:2] == (3, 2), + "python3.2 does not allow u"" literal string definition") + def test_duplicate_keys_bytes_vs_unicode_py3(self): + self.flakes("{b'a': 1, u'a': 2}") + + @skipIf(version_info < (3,), + "bytes and strings with same 'value' are not equal in python3") + @skipIf(version_info[0:2] == (3, 2), + "python3.2 does not allow u"" literal string definition") + def test_duplicate_values_bytes_vs_unicode_py3(self): + self.flakes( + "{1: b'a', 1: u'a'}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + @skipIf(version_info >= (3,), + "bytes and strings with same 'value' are equal in python2") + def test_duplicate_keys_bytes_vs_unicode_py2(self): + self.flakes( + "{b'a': 1, u'a': 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + @skipIf(version_info >= (3,), + "bytes and strings with same 'value' are equal in python2") + def test_duplicate_values_bytes_vs_unicode_py2(self): + self.flakes("{1: b'a', 1: u'a'}") + + def test_multiple_duplicate_keys(self): + self.flakes( + "{'yes': 1, 'yes': 2, 'no': 2, 'no': 3}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_in_function(self): + self.flakes( + ''' + def f(thing): + pass + f({'yes': 1, 'yes': 2}) + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_in_lambda(self): + self.flakes( + "lambda x: {(0,1): 1, (0,1): 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_tuples(self): + self.flakes( + "{(0,1): 1, (0,1): 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_tuples_int_and_float(self): + self.flakes( + "{(0,1): 1, (0,1.0): 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_ints(self): + self.flakes( + "{1: 1, 1: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_bools(self): + self.flakes( + "{True: 1, True: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_bools_false(self): + # Needed to ensure 2.x correctly coerces these from variables + self.flakes( + "{False: 1, False: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_none(self): + self.flakes( + "{None: 1, None: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_variable_keys(self): + self.flakes( + ''' + a = 1 + {a: 1, a: 2} + ''', + m.MultiValueRepeatedKeyVariable, + m.MultiValueRepeatedKeyVariable, + ) + + def test_duplicate_variable_values(self): + self.flakes( + ''' + a = 1 + b = 2 + {1: a, 1: b} + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_variable_values_same_value(self): + # Current behaviour is not to look up variable values. This is to + # confirm that. + self.flakes( + ''' + a = 1 + b = 1 + {1: a, 1: b} + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_key_float_and_int(self): + """ + These do look like different values, but when it comes to their use as + keys, they compare as equal and so are actually duplicates. + The literal dict {1: 1, 1.0: 1} actually becomes {1.0: 1}. + """ + self.flakes( + ''' + {1: 1, 1.0: 2} + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_no_duplicate_key_error_same_value(self): + self.flakes(''' + {'yes': 1, 'yes': 1} + ''') + + def test_no_duplicate_key_errors(self): + self.flakes(''' + {'yes': 1, 'no': 2} + ''') + + def test_no_duplicate_keys_tuples_same_first_element(self): + self.flakes("{(0,1): 1, (0,2): 1}") + + def test_no_duplicate_key_errors_func_call(self): + self.flakes(''' + def test(thing): + pass + test({True: 1, None: 2, False: 1}) + ''') + + def test_no_duplicate_key_errors_bool_or_none(self): + self.flakes("{True: 1, None: 2, False: 1}") + + def test_no_duplicate_key_errors_ints(self): + self.flakes(''' + {1: 1, 2: 1} + ''') + + def test_no_duplicate_key_errors_vars(self): + self.flakes(''' + test = 'yes' + rest = 'yes' + {test: 1, rest: 2} + ''') + + def test_no_duplicate_key_errors_tuples(self): + self.flakes(''' + {(0,1): 1, (0,2): 1} + ''') + + def test_no_duplicate_key_errors_instance_attributes(self): + self.flakes(''' + class Test(): + pass + f = Test() + f.a = 1 + {f.a: 1, f.a: 1} + ''') diff --git a/contrib/pyflakes/test/test_doctests.py b/contrib/pyflakes/test/test_doctests.py index 6a4ba00..aed7957 100644 --- a/contrib/pyflakes/test/test_doctests.py +++ b/contrib/pyflakes/test/test_doctests.py @@ -1,11 +1,23 @@ +import sys import textwrap from pyflakes import messages as m +from pyflakes.checker import ( + DoctestScope, + FunctionScope, + ModuleScope, +) from pyflakes.test.test_other import Test as TestOther from pyflakes.test.test_imports import Test as TestImports from pyflakes.test.test_undefined_names import Test as TestUndefinedNames from pyflakes.test.harness import TestCase, skip +try: + sys.pypy_version_info + PYPY = True +except AttributeError: + PYPY = False + class _DoctestMixin(object): @@ -42,6 +54,190 @@ class Test(TestCase): withDoctest = True + def test_scope_class(self): + """Check that a doctest is given a DoctestScope.""" + checker = self.flakes(""" + m = None + + def doctest_stuff(): + ''' + >>> d = doctest_stuff() + ''' + f = m + return f + """) + + scopes = checker.deadScopes + module_scopes = [ + scope for scope in scopes if scope.__class__ is ModuleScope] + doctest_scopes = [ + scope for scope in scopes if scope.__class__ is DoctestScope] + function_scopes = [ + scope for scope in scopes if scope.__class__ is FunctionScope] + + self.assertEqual(len(module_scopes), 1) + self.assertEqual(len(doctest_scopes), 1) + + module_scope = module_scopes[0] + doctest_scope = doctest_scopes[0] + + self.assertIsInstance(doctest_scope, DoctestScope) + self.assertIsInstance(doctest_scope, ModuleScope) + self.assertNotIsInstance(doctest_scope, FunctionScope) + self.assertNotIsInstance(module_scope, DoctestScope) + + self.assertIn('m', module_scope) + self.assertIn('doctest_stuff', module_scope) + + self.assertIn('d', doctest_scope) + + self.assertEqual(len(function_scopes), 1) + self.assertIn('f', function_scopes[0]) + + def test_nested_doctest_ignored(self): + """Check that nested doctests are ignored.""" + checker = self.flakes(""" + m = None + + def doctest_stuff(): + ''' + >>> def function_in_doctest(): + ... \"\"\" + ... >>> ignored_undefined_name + ... \"\"\" + ... df = m + ... return df + ... + >>> function_in_doctest() + ''' + f = m + return f + """) + + scopes = checker.deadScopes + module_scopes = [ + scope for scope in scopes if scope.__class__ is ModuleScope] + doctest_scopes = [ + scope for scope in scopes if scope.__class__ is DoctestScope] + function_scopes = [ + scope for scope in scopes if scope.__class__ is FunctionScope] + + self.assertEqual(len(module_scopes), 1) + self.assertEqual(len(doctest_scopes), 1) + + module_scope = module_scopes[0] + doctest_scope = doctest_scopes[0] + + self.assertIn('m', module_scope) + self.assertIn('doctest_stuff', module_scope) + self.assertIn('function_in_doctest', doctest_scope) + + self.assertEqual(len(function_scopes), 2) + + self.assertIn('f', function_scopes[0]) + self.assertIn('df', function_scopes[1]) + + def test_global_module_scope_pollution(self): + """Check that global in doctest does not pollute module scope.""" + checker = self.flakes(""" + def doctest_stuff(): + ''' + >>> def function_in_doctest(): + ... global m + ... m = 50 + ... df = 10 + ... m = df + ... + >>> function_in_doctest() + ''' + f = 10 + return f + + """) + + scopes = checker.deadScopes + module_scopes = [ + scope for scope in scopes if scope.__class__ is ModuleScope] + doctest_scopes = [ + scope for scope in scopes if scope.__class__ is DoctestScope] + function_scopes = [ + scope for scope in scopes if scope.__class__ is FunctionScope] + + self.assertEqual(len(module_scopes), 1) + self.assertEqual(len(doctest_scopes), 1) + + module_scope = module_scopes[0] + doctest_scope = doctest_scopes[0] + + self.assertIn('doctest_stuff', module_scope) + self.assertIn('function_in_doctest', doctest_scope) + + self.assertEqual(len(function_scopes), 2) + + self.assertIn('f', function_scopes[0]) + self.assertIn('df', function_scopes[1]) + self.assertIn('m', function_scopes[1]) + + self.assertNotIn('m', module_scope) + + def test_global_undefined(self): + self.flakes(""" + global m + + def doctest_stuff(): + ''' + >>> m + ''' + """, m.UndefinedName) + + def test_nested_class(self): + """Doctest within nested class are processed.""" + self.flakes(""" + class C: + class D: + ''' + >>> m + ''' + def doctest_stuff(self): + ''' + >>> m + ''' + return 1 + """, m.UndefinedName, m.UndefinedName) + + def test_ignore_nested_function(self): + """Doctest module does not process doctest in nested functions.""" + # 'syntax error' would cause a SyntaxError if the doctest was processed. + # However doctest does not find doctest in nested functions + # (https://bugs.python.org/issue1650090). If nested functions were + # processed, this use of m should cause UndefinedName, and the + # name inner_function should probably exist in the doctest scope. + self.flakes(""" + def doctest_stuff(): + def inner_function(): + ''' + >>> syntax error + >>> inner_function() + 1 + >>> m + ''' + return 1 + m = inner_function() + return m + """) + + def test_inaccessible_scope_class(self): + """Doctest may not access class scope.""" + self.flakes(""" + class C: + def doctest_stuff(self): + ''' + >>> m + ''' + return 1 + m = 1 + """, m.UndefinedName) + def test_importBeforeDoctest(self): self.flakes(""" import foo @@ -132,12 +328,22 @@ def doctest_stuff(): exc = exceptions[0] self.assertEqual(exc.lineno, 4) self.assertEqual(exc.col, 26) + + # PyPy error column offset is 0, + # for the second and third line of the doctest + # i.e. at the beginning of the line exc = exceptions[1] self.assertEqual(exc.lineno, 5) - self.assertEqual(exc.col, 16) + if PYPY: + self.assertEqual(exc.col, 13) + else: + self.assertEqual(exc.col, 16) exc = exceptions[2] self.assertEqual(exc.lineno, 6) - self.assertEqual(exc.col, 18) + if PYPY: + self.assertEqual(exc.col, 13) + else: + self.assertEqual(exc.col, 18) def test_indentationErrorInDoctest(self): exc = self.flakes(''' @@ -148,7 +354,10 @@ def doctest_stuff(): """ ''', m.DoctestSyntaxError).messages[0] self.assertEqual(exc.lineno, 5) - self.assertEqual(exc.col, 16) + if PYPY: + self.assertEqual(exc.col, 13) + else: + self.assertEqual(exc.col, 16) def test_offsetWithMultiLineArgs(self): (exc1, exc2) = self.flakes( @@ -222,35 +431,12 @@ def func(): class TestOther(_DoctestMixin, TestOther): - pass + """Run TestOther with each test wrapped in a doctest.""" class TestImports(_DoctestMixin, TestImports): - - def test_futureImport(self): - """XXX This test can't work in a doctest""" - - def test_futureImportUsed(self): - """XXX This test can't work in a doctest""" + """Run TestImports with each test wrapped in a doctest.""" class TestUndefinedNames(_DoctestMixin, TestUndefinedNames): - - def test_doubleNestingReportsClosestName(self): - """ - Lines in doctest are a bit different so we can't use the test - from TestUndefinedNames - """ - exc = self.flakes(''' - def a(): - x = 1 - def b(): - x = 2 # line 7 in the file - def c(): - x - x = 3 - return x - return x - return x - ''', m.UndefinedLocal).messages[0] - self.assertEqual(exc.message_args, ('x', 7)) + """Run TestUndefinedNames with each test wrapped in a doctest.""" diff --git a/contrib/pyflakes/test/test_imports.py b/contrib/pyflakes/test/test_imports.py index 6cd0ec5..3e3be88 100644 --- a/contrib/pyflakes/test/test_imports.py +++ b/contrib/pyflakes/test/test_imports.py @@ -2,26 +2,152 @@ from sys import version_info from pyflakes import messages as m +from pyflakes.checker import ( + FutureImportation, + Importation, + ImportationFrom, + StarImportation, + SubmoduleImportation, +) from pyflakes.test.harness import TestCase, skip, skipIf +class TestImportationObject(TestCase): + + def test_import_basic(self): + binding = Importation('a', None, 'a') + assert binding.source_statement == 'import a' + assert str(binding) == 'a' + + def test_import_as(self): + binding = Importation('c', None, 'a') + assert binding.source_statement == 'import a as c' + assert str(binding) == 'a as c' + + def test_import_submodule(self): + binding = SubmoduleImportation('a.b', None) + assert binding.source_statement == 'import a.b' + assert str(binding) == 'a.b' + + def test_import_submodule_as(self): + # A submodule import with an as clause is not a SubmoduleImportation + binding = Importation('c', None, 'a.b') + assert binding.source_statement == 'import a.b as c' + assert str(binding) == 'a.b as c' + + def test_import_submodule_as_source_name(self): + binding = Importation('a', None, 'a.b') + assert binding.source_statement == 'import a.b as a' + assert str(binding) == 'a.b as a' + + def test_importfrom_relative(self): + binding = ImportationFrom('a', None, '.', 'a') + assert binding.source_statement == 'from . import a' + assert str(binding) == '.a' + + def test_importfrom_relative_parent(self): + binding = ImportationFrom('a', None, '..', 'a') + assert binding.source_statement == 'from .. import a' + assert str(binding) == '..a' + + def test_importfrom_relative_with_module(self): + binding = ImportationFrom('b', None, '..a', 'b') + assert binding.source_statement == 'from ..a import b' + assert str(binding) == '..a.b' + + def test_importfrom_relative_with_module_as(self): + binding = ImportationFrom('c', None, '..a', 'b') + assert binding.source_statement == 'from ..a import b as c' + assert str(binding) == '..a.b as c' + + def test_importfrom_member(self): + binding = ImportationFrom('b', None, 'a', 'b') + assert binding.source_statement == 'from a import b' + assert str(binding) == 'a.b' + + def test_importfrom_submodule_member(self): + binding = ImportationFrom('c', None, 'a.b', 'c') + assert binding.source_statement == 'from a.b import c' + assert str(binding) == 'a.b.c' + + def test_importfrom_member_as(self): + binding = ImportationFrom('c', None, 'a', 'b') + assert binding.source_statement == 'from a import b as c' + assert str(binding) == 'a.b as c' + + def test_importfrom_submodule_member_as(self): + binding = ImportationFrom('d', None, 'a.b', 'c') + assert binding.source_statement == 'from a.b import c as d' + assert str(binding) == 'a.b.c as d' + + def test_importfrom_star(self): + binding = StarImportation('a.b', None) + assert binding.source_statement == 'from a.b import *' + assert str(binding) == 'a.b.*' + + def test_importfrom_star_relative(self): + binding = StarImportation('.b', None) + assert binding.source_statement == 'from .b import *' + assert str(binding) == '.b.*' + + def test_importfrom_future(self): + binding = FutureImportation('print_function', None, None) + assert binding.source_statement == 'from __future__ import print_function' + assert str(binding) == '__future__.print_function' + + class Test(TestCase): def test_unusedImport(self): self.flakes('import fu, bar', m.UnusedImport, m.UnusedImport) self.flakes('from baz import fu, bar', m.UnusedImport, m.UnusedImport) + def test_unusedImport_relative(self): + self.flakes('from . import fu', m.UnusedImport) + self.flakes('from . import fu as baz', m.UnusedImport) + self.flakes('from .. import fu', m.UnusedImport) + self.flakes('from ... import fu', m.UnusedImport) + self.flakes('from .. import fu as baz', m.UnusedImport) + self.flakes('from .bar import fu', m.UnusedImport) + self.flakes('from ..bar import fu', m.UnusedImport) + self.flakes('from ...bar import fu', m.UnusedImport) + self.flakes('from ...bar import fu as baz', m.UnusedImport) + + checker = self.flakes('from . import fu', m.UnusedImport) + + error = checker.messages[0] + assert error.message == '%r imported but unused' + assert error.message_args == ('.fu', ) + + checker = self.flakes('from . import fu as baz', m.UnusedImport) + + error = checker.messages[0] + assert error.message == '%r imported but unused' + assert error.message_args == ('.fu as baz', ) + def test_aliasedImport(self): self.flakes('import fu as FU, bar as FU', m.RedefinedWhileUnused, m.UnusedImport) self.flakes('from moo import fu as FU, bar as FU', m.RedefinedWhileUnused, m.UnusedImport) + def test_aliasedImportShadowModule(self): + """Imported aliases can shadow the source of the import.""" + self.flakes('from moo import fu as moo; moo') + self.flakes('import fu as fu; fu') + self.flakes('import fu.bar as fu; fu') + def test_usedImport(self): self.flakes('import fu; print(fu)') self.flakes('from baz import fu; print(fu)') self.flakes('import fu; del fu') + def test_usedImport_relative(self): + self.flakes('from . import fu; assert fu') + self.flakes('from .bar import fu; assert fu') + self.flakes('from .. import fu; assert fu') + self.flakes('from ..bar import fu as baz; assert baz') + def test_redefinedWhileUnused(self): self.flakes('import fu; fu = 3', m.RedefinedWhileUnused) self.flakes('import fu; fu, bar = 3', m.RedefinedWhileUnused) @@ -54,7 +180,7 @@ def test_redefinedIfElse(self): def test_redefinedTry(self): """ - Test that importing a module twice in an try block + Test that importing a module twice in a try block does raise a warning. """ self.flakes(''' @@ -67,7 +193,7 @@ def test_redefinedTry(self): def test_redefinedTryExcept(self): """ - Test that importing a module twice in an try + Test that importing a module twice in a try and except block does not raise a warning. """ self.flakes(''' @@ -236,6 +362,22 @@ class bar: print(fu) ''') + def test_importInClass(self): + """ + Test that import within class is a locally scoped attribute. + """ + self.flakes(''' + class bar: + import fu + ''') + + self.flakes(''' + class bar: + import fu + + fu + ''', m.UndefinedName) + def test_usedInFunction(self): self.flakes(''' import fu @@ -570,7 +712,7 @@ class bar: import fu def fun(self): fu - ''', m.UnusedImport, m.UndefinedName) + ''', m.UndefinedName) def test_nestedFunctionsNestScope(self): self.flakes(''' @@ -590,7 +732,89 @@ def c(self): ''') def test_importStar(self): - self.flakes('from fu import *', m.ImportStarUsed) + """Use of import * at module level is reported.""" + self.flakes('from fu import *', m.ImportStarUsed, m.UnusedImport) + self.flakes(''' + try: + from fu import * + except: + pass + ''', m.ImportStarUsed, m.UnusedImport) + + checker = self.flakes('from fu import *', + m.ImportStarUsed, m.UnusedImport) + + error = checker.messages[0] + assert error.message.startswith("'from %s import *' used; unable ") + assert error.message_args == ('fu', ) + + error = checker.messages[1] + assert error.message == '%r imported but unused' + assert error.message_args == ('fu.*', ) + + def test_importStar_relative(self): + """Use of import * from a relative import is reported.""" + self.flakes('from .fu import *', m.ImportStarUsed, m.UnusedImport) + self.flakes(''' + try: + from .fu import * + except: + pass + ''', m.ImportStarUsed, m.UnusedImport) + + checker = self.flakes('from .fu import *', + m.ImportStarUsed, m.UnusedImport) + + error = checker.messages[0] + assert error.message.startswith("'from %s import *' used; unable ") + assert error.message_args == ('.fu', ) + + error = checker.messages[1] + assert error.message == '%r imported but unused' + assert error.message_args == ('.fu.*', ) + + checker = self.flakes('from .. import *', + m.ImportStarUsed, m.UnusedImport) + + error = checker.messages[0] + assert error.message.startswith("'from %s import *' used; unable ") + assert error.message_args == ('..', ) + + error = checker.messages[1] + assert error.message == '%r imported but unused' + assert error.message_args == ('from .. import *', ) + + @skipIf(version_info < (3,), + 'import * below module level is a warning on Python 2') + def test_localImportStar(self): + """import * is only allowed at module level.""" + self.flakes(''' + def a(): + from fu import * + ''', m.ImportStarNotPermitted) + self.flakes(''' + class a: + from fu import * + ''', m.ImportStarNotPermitted) + + checker = self.flakes(''' + class a: + from .. import * + ''', m.ImportStarNotPermitted) + error = checker.messages[0] + assert error.message == "'from %s import *' only allowed at module level" + assert error.message_args == ('..', ) + + @skipIf(version_info > (3,), + 'import * below module level is an error on Python 3') + def test_importStarNested(self): + """All star imports are marked as used by an undefined variable.""" + self.flakes(''' + from fu import * + def f(): + from bar import * + x + ''', m.ImportStarUsed, m.ImportStarUsed, m.ImportStarUsage) def test_packageImport(self): """ @@ -638,6 +862,35 @@ def test_differentSubmoduleImport(self): fu.bar, fu.baz ''') + def test_used_package_with_submodule_import(self): + """ + Usage of package marks submodule imports as used. + """ + self.flakes(''' + import fu + import fu.bar + fu.x + ''') + + self.flakes(''' + import fu.bar + import fu + fu.x + ''') + + def test_unused_package_with_submodule_import(self): + """ + When a package and its submodule are imported, only report once. + """ + checker = self.flakes(''' + import fu + import fu.bar + ''', m.UnusedImport) + error = checker.messages[0] + assert error.message == '%r imported but unused' + assert error.message_args == ('fu.bar', ) + assert error.lineno == 5 if self.withDoctest else 3 + def test_assignRHSFirst(self): self.flakes('import fu; fu = fu') self.flakes('import fu; fu, bar = fu') @@ -689,7 +942,6 @@ def test_importingForImportError(self): pass ''') - @skip("todo: requires evaluating attribute access") def test_importedInClass(self): """Imports in class scope can be used through self.""" self.flakes(''' @@ -742,6 +994,18 @@ def test_futureImportUsed(self): assert print_function is not division ''') + def test_futureImportUndefined(self): + """Importing undefined names from __future__ fails.""" + self.flakes(''' + from __future__ import print_statement + ''', m.FutureFeatureNotDefined) + + def test_futureImportStar(self): + """Importing '*' from __future__ fails.""" + self.flakes(''' + from __future__ import * + ''', m.FutureFeatureNotDefined) + class TestSpecialAll(TestCase): """ @@ -760,12 +1024,11 @@ def foo(): def test_ignoredInClass(self): """ - An C{__all__} definition does not suppress unused import warnings in a - class scope. + An C{__all__} definition in a class does not suppress unused import warnings. """ self.flakes(''' + import bar class foo: - import bar __all__ = ["bar"] ''', m.UnusedImport) @@ -834,6 +1097,14 @@ def test_importStarExported(self): __all__ = ["foo"] ''', m.ImportStarUsed) + def test_importStarNotExported(self): + """Report unused import when not needed to satisfy __all__.""" + self.flakes(''' + from foolib import * + a = 1 + __all__ = ['a'] + ''', m.ImportStarUsed, m.UnusedImport) + def test_usedInGenExp(self): """ Using a global in a generator expression results in no warnings. @@ -878,7 +1149,7 @@ def f(): class Python26Tests(TestCase): """ - Tests for checking of syntax which is valid in PYthon 2.6 and newer. + Tests for checking of syntax which is valid in Python 2.6 and newer. """ @skipIf(version_info < (2, 6), "Python >= 2.6 only") diff --git a/contrib/pyflakes/test/test_other.py b/contrib/pyflakes/test/test_other.py index 384c4b3..9c8462e 100644 --- a/contrib/pyflakes/test/test_other.py +++ b/contrib/pyflakes/test/test_other.py @@ -353,6 +353,664 @@ class Foo(object): return ''', m.ReturnOutsideFunction) + def test_moduleWithReturn(self): + """ + If a return is used at the module level, a warning is emitted. + """ + self.flakes(''' + return + ''', m.ReturnOutsideFunction) + + def test_classWithYield(self): + """ + If a yield is used inside a class, a warning is emitted. + """ + self.flakes(''' + class Foo(object): + yield + ''', m.YieldOutsideFunction) + + def test_moduleWithYield(self): + """ + If a yield is used at the module level, a warning is emitted. + """ + self.flakes(''' + yield + ''', m.YieldOutsideFunction) + + @skipIf(version_info < (3, 3), "Python >= 3.3 only") + def test_classWithYieldFrom(self): + """ + If a yield from is used inside a class, a warning is emitted. + """ + self.flakes(''' + class Foo(object): + yield from range(10) + ''', m.YieldOutsideFunction) + + @skipIf(version_info < (3, 3), "Python >= 3.3 only") + def test_moduleWithYieldFrom(self): + """ + If a yield from is used at the module level, a warning is emitted. + """ + self.flakes(''' + yield from range(10) + ''', m.YieldOutsideFunction) + + def test_continueOutsideLoop(self): + self.flakes(''' + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + def f(): + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + pass + else: + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + pass + else: + if 1: + if 2: + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + def f(): + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + class A: + continue + ''', m.ContinueOutsideLoop) + + def test_continueInsideLoop(self): + self.flakes(''' + while True: + continue + ''') + + self.flakes(''' + for i in range(10): + continue + ''') + + self.flakes(''' + while True: + if 1: + continue + ''') + + self.flakes(''' + for i in range(10): + if 1: + continue + ''') + + self.flakes(''' + while True: + while True: + pass + else: + continue + else: + pass + ''') + + self.flakes(''' + while True: + try: + pass + finally: + while True: + continue + ''') + + def test_continueInFinally(self): + # 'continue' inside 'finally' is a special syntax error + self.flakes(''' + while True: + try: + pass + finally: + continue + ''', m.ContinueInFinally) + + self.flakes(''' + while True: + try: + pass + finally: + if 1: + if 2: + continue + ''', m.ContinueInFinally) + + # Even when not in a loop, this is the error Python gives + self.flakes(''' + try: + pass + finally: + continue + ''', m.ContinueInFinally) + + def test_breakOutsideLoop(self): + self.flakes(''' + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + def f(): + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + pass + else: + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + pass + else: + if 1: + if 2: + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + def f(): + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + class A: + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + try: + pass + finally: + break + ''', m.BreakOutsideLoop) + + def test_breakInsideLoop(self): + self.flakes(''' + while True: + break + ''') + + self.flakes(''' + for i in range(10): + break + ''') + + self.flakes(''' + while True: + if 1: + break + ''') + + self.flakes(''' + for i in range(10): + if 1: + break + ''') + + self.flakes(''' + while True: + while True: + pass + else: + break + else: + pass + ''') + + self.flakes(''' + while True: + try: + pass + finally: + while True: + break + ''') + + self.flakes(''' + while True: + try: + pass + finally: + break + ''') + + self.flakes(''' + while True: + try: + pass + finally: + if 1: + if 2: + break + ''') + + def test_defaultExceptLast(self): + """ + A default except block should be last. + + YES: + + try: + ... + except Exception: + ... + except: + ... + + NO: + + try: + ... + except: + ... + except Exception: + ... + """ + self.flakes(''' + try: + pass + except ValueError: + pass + ''') + + self.flakes(''' + try: + pass + except ValueError: + pass + except: + pass + ''') + + self.flakes(''' + try: + pass + except: + pass + ''') + + self.flakes(''' + try: + pass + except ValueError: + pass + else: + pass + ''') + + self.flakes(''' + try: + pass + except: + pass + else: + pass + ''') + + self.flakes(''' + try: + pass + except ValueError: + pass + except: + pass + else: + pass + ''') + + def test_defaultExceptNotLast(self): + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + else: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + else: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + else: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + else: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + finally: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + @skipIf(version_info < (3,), "Python 3 only") + def test_starredAssignmentNoError(self): + """ + Python 3 extended iterable unpacking + """ + self.flakes(''' + a, *b = range(10) + ''') + + self.flakes(''' + *a, b = range(10) + ''') + + self.flakes(''' + a, *b, c = range(10) + ''') + + self.flakes(''' + (a, *b) = range(10) + ''') + + self.flakes(''' + (*a, b) = range(10) + ''') + + self.flakes(''' + (a, *b, c) = range(10) + ''') + + self.flakes(''' + [a, *b] = range(10) + ''') + + self.flakes(''' + [*a, b] = range(10) + ''') + + self.flakes(''' + [a, *b, c] = range(10) + ''') + + # Taken from test_unpack_ex.py in the cPython source + s = ", ".join("a%d" % i for i in range(1 << 8 - 1)) + \ + ", *rest = range(1<<8)" + self.flakes(s) + + s = "(" + ", ".join("a%d" % i for i in range(1 << 8 - 1)) + \ + ", *rest) = range(1<<8)" + self.flakes(s) + + s = "[" + ", ".join("a%d" % i for i in range(1 << 8 - 1)) + \ + ", *rest] = range(1<<8)" + self.flakes(s) + + @skipIf(version_info < (3, ), "Python 3 only") + def test_starredAssignmentErrors(self): + """ + SyntaxErrors (not encoded in the ast) surrounding Python 3 extended + iterable unpacking + """ + # Taken from test_unpack_ex.py in the cPython source + s = ", ".join("a%d" % i for i in range(1 << 8)) + \ + ", *rest = range(1<<8 + 1)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "(" + ", ".join("a%d" % i for i in range(1 << 8)) + \ + ", *rest) = range(1<<8 + 1)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "[" + ", ".join("a%d" % i for i in range(1 << 8)) + \ + ", *rest] = range(1<<8 + 1)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = ", ".join("a%d" % i for i in range(1 << 8 + 1)) + \ + ", *rest = range(1<<8 + 2)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "(" + ", ".join("a%d" % i for i in range(1 << 8 + 1)) + \ + ", *rest) = range(1<<8 + 2)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "[" + ", ".join("a%d" % i for i in range(1 << 8 + 1)) + \ + ", *rest] = range(1<<8 + 2)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + # No way we can actually test this! + # s = "*rest, " + ", ".join("a%d" % i for i in range(1<<24)) + \ + # ", *rest = range(1<<24 + 1)" + # self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + self.flakes(''' + a, *b, *c = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + a, *b, c, *d = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + *a, *b, *c = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + (a, *b, *c) = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + (a, *b, c, *d) = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + (*a, *b, *c) = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + [a, *b, *c] = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + [a, *b, c, *d] = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + [*a, *b, *c] = range(10) + ''', m.TwoStarredExpressions) + @skip("todo: Too hard to make this warn but other cases stay silent") def test_doubleAssignment(self): """ @@ -523,7 +1181,7 @@ def a(): return ''', m.UnusedVariable) - @skip("todo: Difficult because it does't apply in the context of a loop") + @skip("todo: Difficult because it doesn't apply in the context of a loop") def test_unusedReassignedVariable(self): """ Shadowing a used variable can still raise an UnusedVariable warning. @@ -831,7 +1489,7 @@ def test_withStatementSingleNameUndefined(self): def test_withStatementTupleNamesUndefined(self): """ - An undefined name warning is emitted if a name first defined by a the + An undefined name warning is emitted if a name first defined by the tuple-unpacking form of the C{with} statement is used before the C{with} statement. """ @@ -975,6 +1633,40 @@ def test_augmentedAssignmentImportedFunctionCall(self): baz += bar() ''') + def test_assert_without_message(self): + """An assert without a message is not an error.""" + self.flakes(''' + a = 1 + assert a + ''') + + def test_assert_with_message(self): + """An assert with a message is not an error.""" + self.flakes(''' + a = 1 + assert a, 'x' + ''') + + def test_assert_tuple(self): + """An assert of a non-empty tuple is always True.""" + self.flakes(''' + assert (False, 'x') + assert (False, ) + ''', m.AssertTuple, m.AssertTuple) + + def test_assert_tuple_empty(self): + """An assert of an empty tuple is always False.""" + self.flakes(''' + assert () + ''') + + def test_assert_static(self): + """An assert of a static value is not an error.""" + self.flakes(''' + assert True + assert 1 + ''') + @skipIf(version_info < (3, 3), 'new in Python 3.3') def test_yieldFromUndefined(self): """ @@ -985,9 +1677,13 @@ def bar(): yield from foo() ''', m.UndefinedName) - def test_returnOnly(self): - """Do not crash on lone "return".""" - self.flakes('return 2') + @skipIf(version_info < (3, 6), 'new in Python 3.6') + def test_f_string(self): + """Test PEP 498 f-strings are treated as a usage.""" + self.flakes(''' + baz = 0 + print(f'\x7b4*baz\N{RIGHT CURLY BRACKET}') + ''') class TestAsyncStatements(TestCase): @@ -1023,6 +1719,63 @@ async def read_data(db): return output ''') + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_loopControlInAsyncFor(self): + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + if row[0] == 'skip': + continue + output.append(row) + return output + ''') + + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + if row[0] == 'stop': + break + output.append(row) + return output + ''') + + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_loopControlInAsyncForElse(self): + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + output.append(row) + else: + continue + return output + ''', m.ContinueOutsideLoop) + + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + output.append(row) + else: + break + return output + ''', m.BreakOutsideLoop) + + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_continueInAsyncForFinally(self): + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + try: + output.append(row) + finally: + continue + return output + ''', m.ContinueInFinally) + @skipIf(version_info < (3, 5), 'new in Python 3.5') def test_asyncWith(self): self.flakes(''' @@ -1040,3 +1793,84 @@ async def commit(session, data): ... await trans.end() ''') + + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_matmul(self): + self.flakes(''' + def foo(a, b): + return a @ b + ''') + + @skipIf(version_info < (3, 6), 'new in Python 3.6') + def test_formatstring(self): + self.flakes(''' + hi = 'hi' + mom = 'mom' + f'{hi} {mom}' + ''') + + @skipIf(version_info < (3, 6), 'new in Python 3.6') + def test_variable_annotations(self): + self.flakes(''' + name: str + age: int + ''') + self.flakes(''' + name: str = 'Bob' + age: int = 18 + ''') + self.flakes(''' + class C: + name: str + age: int + ''') + self.flakes(''' + class C: + name: str = 'Bob' + age: int = 18 + ''') + self.flakes(''' + def f(): + name: str + age: int + ''') + self.flakes(''' + def f(): + name: str = 'Bob' + age: int = 18 + foo: not_a_real_type = None + ''', m.UnusedVariable, m.UnusedVariable, m.UnusedVariable, m.UndefinedName) + self.flakes(''' + def f(): + name: str + print(name) + ''', m.UndefinedName) + self.flakes(''' + from typing import Any + def f(): + a: Any + ''') + self.flakes(''' + foo: not_a_real_type + ''', m.UndefinedName) + self.flakes(''' + foo: not_a_real_type = None + ''', m.UndefinedName) + self.flakes(''' + class C: + foo: not_a_real_type + ''', m.UndefinedName) + self.flakes(''' + class C: + foo: not_a_real_type = None + ''', m.UndefinedName) + self.flakes(''' + def f(): + class C: + foo: not_a_real_type + ''', m.UndefinedName) + self.flakes(''' + def f(): + class C: + foo: not_a_real_type = None + ''', m.UndefinedName) diff --git a/contrib/pyflakes/test/test_undefined_names.py b/contrib/pyflakes/test/test_undefined_names.py index faaaf8c..1464d8e 100644 --- a/contrib/pyflakes/test/test_undefined_names.py +++ b/contrib/pyflakes/test/test_undefined_names.py @@ -3,7 +3,7 @@ from sys import version_info from pyflakes import messages as m, checker -from pyflakes.test.harness import TestCase, skipIf +from pyflakes.test.harness import TestCase, skipIf, skip class Test(TestCase): @@ -22,6 +22,184 @@ def test_undefinedInListComp(self): ''', m.UndefinedName) + @skipIf(version_info < (3,), + 'in Python 2 exception names stay bound after the except: block') + def test_undefinedExceptionName(self): + """Exception names can't be used after the except: block.""" + self.flakes(''' + try: + raise ValueError('ve') + except ValueError as exc: + pass + exc + ''', + m.UndefinedName) + + def test_namesDeclaredInExceptBlocks(self): + """Locals declared in except: blocks can be used after the block. + + This shows the example in test_undefinedExceptionName is + different.""" + self.flakes(''' + try: + raise ValueError('ve') + except ValueError as exc: + e = exc + e + ''') + + @skip('error reporting disabled due to false positives below') + def test_undefinedExceptionNameObscuringLocalVariable(self): + """Exception names obscure locals, can't be used after. + + Last line will raise UnboundLocalError on Python 3 after exiting + the except: block. Note next two examples for false positives to + watch out for.""" + self.flakes(''' + exc = 'Original value' + try: + raise ValueError('ve') + except ValueError as exc: + pass + exc + ''', + m.UndefinedName) + + @skipIf(version_info < (3,), + 'in Python 2 exception names stay bound after the except: block') + def test_undefinedExceptionNameObscuringLocalVariable2(self): + """Exception names are unbound after the `except:` block. + + Last line will raise UnboundLocalError on Python 3 but would print out + 've' on Python 2.""" + self.flakes(''' + try: + raise ValueError('ve') + except ValueError as exc: + pass + print(exc) + exc = 'Original value' + ''', + m.UndefinedName) + + def test_undefinedExceptionNameObscuringLocalVariableFalsePositive1(self): + """Exception names obscure locals, can't be used after. Unless. + + Last line will never raise UnboundLocalError because it's only + entered if no exception was raised.""" + self.flakes(''' + exc = 'Original value' + try: + raise ValueError('ve') + except ValueError as exc: + print('exception logged') + raise + exc + ''') + + def test_delExceptionInExcept(self): + """The exception name can be deleted in the except: block.""" + self.flakes(''' + try: + pass + except Exception as exc: + del exc + ''') + + def test_undefinedExceptionNameObscuringLocalVariableFalsePositive2(self): + """Exception names obscure locals, can't be used after. Unless. + + Last line will never raise UnboundLocalError because `error` is + only falsy if the `except:` block has not been entered.""" + self.flakes(''' + exc = 'Original value' + error = None + try: + raise ValueError('ve') + except ValueError as exc: + error = 'exception logged' + if error: + print(error) + else: + exc + ''') + + @skip('error reporting disabled due to false positives below') + def test_undefinedExceptionNameObscuringGlobalVariable(self): + """Exception names obscure globals, can't be used after. + + Last line will raise UnboundLocalError on both Python 2 and + Python 3 because the existence of that exception name creates + a local scope placeholder for it, obscuring any globals, etc.""" + self.flakes(''' + exc = 'Original value' + def func(): + try: + pass # nothing is raised + except ValueError as exc: + pass # block never entered, exc stays unbound + exc + ''', + m.UndefinedLocal) + + @skip('error reporting disabled due to false positives below') + def test_undefinedExceptionNameObscuringGlobalVariable2(self): + """Exception names obscure globals, can't be used after. + + Last line will raise NameError on Python 3 because the name is + locally unbound after the `except:` block, even if it's + nonlocal. We should issue an error in this case because code + only working correctly if an exception isn't raised, is invalid. + Unless it's explicitly silenced, see false positives below.""" + self.flakes(''' + exc = 'Original value' + def func(): + global exc + try: + raise ValueError('ve') + except ValueError as exc: + pass # block never entered, exc stays unbound + exc + ''', + m.UndefinedLocal) + + def test_undefinedExceptionNameObscuringGlobalVariableFalsePositive1(self): + """Exception names obscure globals, can't be used after. Unless. + + Last line will never raise NameError because it's only entered + if no exception was raised.""" + self.flakes(''' + exc = 'Original value' + def func(): + global exc + try: + raise ValueError('ve') + except ValueError as exc: + print('exception logged') + raise + exc + ''') + + def test_undefinedExceptionNameObscuringGlobalVariableFalsePositive2(self): + """Exception names obscure globals, can't be used after. Unless. + + Last line will never raise NameError because `error` is only + falsy if the `except:` block has not been entered.""" + self.flakes(''' + exc = 'Original value' + def func(): + global exc + error = None + try: + raise ValueError('ve') + except ValueError as exc: + error = 'exception logged' + if error: + print(error) + else: + exc + ''') + def test_functionsNeedGlobalScope(self): self.flakes(''' class a: @@ -71,8 +249,10 @@ def test_magicGlobalsPath(self): def test_globalImportStar(self): """Can't find undefined names with import *.""" - self.flakes('from fu import *; bar', m.ImportStarUsed) + self.flakes('from fu import *; bar', + m.ImportStarUsed, m.ImportStarUsage) + @skipIf(version_info >= (3,), 'obsolete syntax') def test_localImportStar(self): """ A local import * still allows undefined names to be found @@ -82,7 +262,7 @@ def test_localImportStar(self): def a(): from fu import * bar - ''', m.ImportStarUsed, m.UndefinedName) + ''', m.ImportStarUsed, m.UndefinedName, m.UnusedImport) @skipIf(version_info >= (3,), 'obsolete syntax') def test_unpackedParameter(self): @@ -125,6 +305,29 @@ def foo(): print(x) ''', m.UndefinedName) + def test_global_reset_name_only(self): + """A global statement does not prevent other names being undefined.""" + # Only different undefined names are reported. + # See following test that fails where the same name is used. + self.flakes(''' + def f1(): + s + + def f2(): + global m + ''', m.UndefinedName) + + @skip("todo") + def test_unused_global(self): + """An unused global statement does not define the name.""" + self.flakes(''' + def f1(): + m + + def f2(): + global m + ''', m.UndefinedName) + def test_del(self): """Del deletes bindings.""" self.flakes('a = 1; del a; a', m.UndefinedName) @@ -286,7 +489,11 @@ def c(): return x return x ''', m.UndefinedLocal).messages[0] - self.assertEqual(exc.message_args, ('x', 5)) + + # _DoctestMixin.flakes adds two lines preceding the code above. + expected_line_num = 7 if self.withDoctest else 5 + + self.assertEqual(exc.message_args, ('x', expected_line_num)) def test_laterRedefinedGlobalFromNestedScope3(self): """ @@ -453,8 +660,20 @@ def test_definedInGenExp(self): Using the loop variable of a generator expression results in no warnings. """ - self.flakes('(a for a in %srange(10) if a)' % - ('x' if version_info < (3,) else '')) + self.flakes('(a for a in [1, 2, 3] if a)') + + self.flakes('(b for b in (a for a in [1, 2, 3] if a) if b)') + + def test_undefinedInGenExpNested(self): + """ + The loop variables of generator expressions nested together are + not defined in the other generator. + """ + self.flakes('(b for b in (a for a in [1, 2, 3] if b) if b)', + m.UndefinedName) + + self.flakes('(b for b in (a for a in [1, 2, 3] if a) if a)', + m.UndefinedName) def test_undefinedWithErrorHandler(self): """ @@ -509,6 +728,15 @@ class A: Y = {x:x for x in T} ''') + def test_definedInClassNested(self): + """Defined name for nested generator expressions in a class.""" + self.flakes(''' + class A: + T = range(10) + + Z = (x for x in (a for a in T)) + ''') + def test_undefinedInLoop(self): """ The loop variable is defined after the expression is computed. diff --git a/contrib/six.py b/contrib/six.py new file mode 100644 index 0000000..6bf4fd3 --- /dev/null +++ b/contrib/six.py @@ -0,0 +1,891 @@ +# Copyright (c) 2010-2017 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Utilities for writing code that runs on Python 2 and 3""" + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.11.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") + + +if sys.version_info[:2] == (3, 2): + exec_("""def raise_from(value, from_value): + try: + if from_value is None: + raise value + raise value from from_value + finally: + value = None +""") +elif sys.version_info[:2] > (3, 2): + exec_("""def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + def wrapper(f): + f = functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + return wrapper +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(type): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/contrib/snowballstemmer/__init__.py b/contrib/snowballstemmer/__init__.py new file mode 100644 index 0000000..e0ea2e7 --- /dev/null +++ b/contrib/snowballstemmer/__init__.py @@ -0,0 +1,57 @@ +__all__ = ('language', 'stemmer') + +from .danish_stemmer import DanishStemmer +from .dutch_stemmer import DutchStemmer +from .english_stemmer import EnglishStemmer +from .finnish_stemmer import FinnishStemmer +from .french_stemmer import FrenchStemmer +from .german_stemmer import GermanStemmer +from .hungarian_stemmer import HungarianStemmer +from .italian_stemmer import ItalianStemmer +from .norwegian_stemmer import NorwegianStemmer +from .porter_stemmer import PorterStemmer +from .portuguese_stemmer import PortugueseStemmer +from .romanian_stemmer import RomanianStemmer +from .russian_stemmer import RussianStemmer +from .spanish_stemmer import SpanishStemmer +from .swedish_stemmer import SwedishStemmer +from .turkish_stemmer import TurkishStemmer + +_languages = { + 'danish': DanishStemmer, + 'dutch': DutchStemmer, + 'english': EnglishStemmer, + 'finnish': FinnishStemmer, + 'french': FrenchStemmer, + 'german': GermanStemmer, + 'hungarian': HungarianStemmer, + 'italian': ItalianStemmer, + 'norwegian': NorwegianStemmer, + 'porter': PorterStemmer, + 'portuguese': PortugueseStemmer, + 'romanian': RomanianStemmer, + 'russian': RussianStemmer, + 'spanish': SpanishStemmer, + 'swedish': SwedishStemmer, + 'turkish': TurkishStemmer, +} + +try: + import Stemmer + cext_available = True +except ImportError: + cext_available = False + +def algorithms(): + if cext_available: + return Stemmer.language() + else: + return list(_languages.keys()) + +def stemmer(lang): + if cext_available: + return Stemmer.Stemmer(lang) + if lang.lower() in _languages: + return _languages[lang.lower()]() + else: + raise KeyError("Stemming algorithm '%s' not found" % lang) diff --git a/contrib/snowballstemmer/among.py b/contrib/snowballstemmer/among.py new file mode 100644 index 0000000..5a99ad2 --- /dev/null +++ b/contrib/snowballstemmer/among.py @@ -0,0 +1,15 @@ + +class Among(object): + def __init__(self, s, substring_i, result, method=None): + """ + @ivar s_size search string size + @ivar s search string + @ivar substring index to longest matching substring + @ivar result of the lookup + @ivar method method to use if substring matches + """ + self.s_size = len(s) + self.s = s + self.substring_i = substring_i + self.result = result + self.method = method diff --git a/contrib/snowballstemmer/basestemmer.py b/contrib/snowballstemmer/basestemmer.py new file mode 100644 index 0000000..d7ed09b --- /dev/null +++ b/contrib/snowballstemmer/basestemmer.py @@ -0,0 +1,351 @@ +class BaseStemmer(object): + def __init__(self): + self.set_current("") + self.maxCacheSize = 10000 + self._cache = {} + self._counter = 0 + + def set_current(self, value): + ''' + Set the self.current string. + ''' + self.current = value + self.cursor = 0 + self.limit = len(self.current) + self.limit_backward = 0 + self.bra = self.cursor + self.ket = self.limit + + def get_current(self): + ''' + Get the self.current string. + ''' + return self.current + + def copy_from(self, other): + self.current = other.current + self.cursor = other.cursor + self.limit = other.limit + self.limit_backward = other.limit_backward + self.bra = other.bra + self.ket = other.ket + + def in_grouping(self, s, min, max): + if self.cursor >= self.limit: + return False + ch = ord(self.current[self.cursor]) + if ch > max or ch < min: + return False + ch -= min + if (s[ch >> 3] & (0x1 << (ch & 0x7))) == 0: + return False + self.cursor += 1 + return True + + def in_grouping_b(self, s, min, max): + if self.cursor <= self.limit_backward: + return False + ch = ord(self.current[self.cursor - 1]) + if ch > max or ch < min: + return False + ch -= min + if (s[ch >> 3] & (0x1 << (ch & 0x7))) == 0: + return False + self.cursor -= 1 + return True + + def out_grouping(self, s, min, max): + if self.cursor >= self.limit: + return False + ch = ord(self.current[self.cursor]) + if ch > max or ch < min: + self.cursor += 1 + return True + ch -= min + if (s[ch >> 3] & (0X1 << (ch & 0x7))) == 0: + self.cursor += 1 + return True + return False + + def out_grouping_b(self, s, min, max): + if self.cursor <= self.limit_backward: + return False + ch = ord(self.current[self.cursor - 1]) + if ch > max or ch < min: + self.cursor -= 1 + return True + ch -= min + if (s[ch >> 3] & (0X1 << (ch & 0x7))) == 0: + self.cursor -= 1 + return True + return False + + def in_range(self, min, max): + if self.cursor >= self.limit: + return False + ch = ord(self.current[self.cursor]) + if ch > max or ch < min: + return False + self.cursor += 1 + return True + + def in_range_b(self, min, max): + if self.cursor <= self.limit_backward: + return False + ch = ord(self.current[self.cursor - 1]) + if ch > max or ch < min: + return False + self.cursor -= 1 + return True + + def out_range(self, min, max): + if self.cursor >= self.limit: + return False + ch = ord(self.current[self.cursor]) + if not (ch > max or ch < min): + return False + self.cursor += 1 + return True + + def out_range_b(self, min, max): + if self.cursor <= self.limit_backward: + return False + ch = ord(self.current[self.cursor - 1]) + if not (ch > max or ch < min): + return False + self.cursor -= 1 + return True + + def eq_s(self, s_size, s): + if self.limit - self.cursor < s_size: + return False + if self.current[self.cursor:self.cursor + s_size] != s: + return False + self.cursor += s_size + return True + + def eq_s_b(self, s_size, s): + if self.cursor - self.limit_backward < s_size: + return False + if self.current[self.cursor - s_size:self.cursor] != s: + return False + self.cursor -= s_size + return True + + def eq_v(self, s): + return self.eq_s(len(s), s) + + def eq_v_b(self, s): + return self.eq_s_b(len(s), s) + + def find_among(self, v, v_size): + i = 0 + j = v_size + + c = self.cursor + l = self.limit + + common_i = 0 + common_j = 0 + + first_key_inspected = False + + while True: + k = i + ((j - i) >> 1) + diff = 0 + common = min(common_i, common_j) # smalle + w = v[k] + for i2 in range(common, w.s_size): + if c + common == l: + diff = -1 + break + diff = ord(self.current[c + common]) - ord(w.s[i2]) + if diff != 0: + break + common += 1 + if diff < 0: + j = k + common_j = common + else: + i = k + common_i = common + if j - i <= 1: + if i > 0: + break # v->s has been inspected + if j == i: + break # only one item in v + # - but now we need to go round once more to get + # v->s inspected. self looks messy, but is actually + # the optimal approach. + if first_key_inspected: + break + first_key_inspected = True + while True: + w = v[i] + if common_i >= w.s_size: + self.cursor = c + w.s_size + if w.method is None: + return w.result + method = getattr(self, w.method) + res = method() + self.cursor = c + w.s_size + if res: + return w.result + i = w.substring_i + if i < 0: + return 0 + return -1 # not reachable + + def find_among_b(self, v, v_size): + ''' + find_among_b is for backwards processing. Same comments apply + ''' + i = 0 + j = v_size + + c = self.cursor + lb = self.limit_backward; + + common_i = 0 + common_j = 0 + + first_key_inspected = False + + while True: + k = i + ((j - i) >> 1) + diff = 0 + common = min(common_i, common_j) + w = v[k] + for i2 in range(w.s_size - 1 - common, -1, -1): + if c - common == lb: + diff = -1 + break + diff = ord(self.current[c - 1 - common]) - ord(w.s[i2]) + if diff != 0: + break + common += 1 + if diff < 0: + j = k + common_j = common + else: + i = k + common_i = common + if j - i <= 1: + if i > 0: + break + if j == i: + break + if first_key_inspected: + break + first_key_inspected = True + while True: + w = v[i] + if common_i >= w.s_size: + self.cursor = c - w.s_size + if w.method is None: + return w.result + method = getattr(self, w.method) + res = method() + self.cursor = c - w.s_size + if res: + return w.result + i = w.substring_i + if i < 0: + return 0 + return -1 # not reachable + + def replace_s(self, c_bra, c_ket, s): + ''' + to replace chars between c_bra and c_ket in self.current by the + chars in s. + + @type c_bra int + @type c_ket int + @type s: string + ''' + adjustment = len(s) - (c_ket - c_bra) + self.current = self.current[0:c_bra] + s + self.current[c_ket:] + self.limit += adjustment + if self.cursor >= c_ket: + self.cursor += adjustment + elif self.cursor > c_bra: + self.cursor = c_bra + return adjustment + + def slice_check(self): + if self.bra < 0 or self.bra > self.ket or self.ket > self.limit or self.limit > len(self.current): + return False + return True + + def slice_from(self, s): + ''' + @type s string + ''' + result = False + if self.slice_check(): + self.replace_s(self.bra, self.ket, s) + result = True + return result + + def slice_del(self): + return self.slice_from("") + + def insert(self, c_bra, c_ket, s): + ''' + @type c_bra int + @type c_ket int + @type s: string + ''' + adjustment = self.replace_s(c_bra, c_ket, s) + if c_bra <= self.bra: + self.bra += adjustment + if c_bra <= self.ket: + self.ket += adjustment + + def slice_to(self, s): + ''' + Copy the slice into the supplied StringBuffer + + @type s: string + ''' + result = '' + if self.slice_check(): + result = self.current[self.bra:self.ket] + return result + + def assign_to(self, s): + ''' + @type s: string + ''' + return self.current[0:self.limit] + + def _stem_word(self, word): + cache = self._cache.get(word) + if cache is None: + self.set_current(word) + self._stem() + result = self.get_current() + self._cache[word] = [result, self._counter] + else: + cache[1] = self._counter + result = cache[0] + self._counter += 1 + return result + + def _clear_cache(self): + removecount = int(len(self._cache) - self.maxCacheSize * 8 / 10) + oldcaches = sorted(self._cache.items(), key=lambda cache: cache[1][1])[0:removecount] + for key, value in oldcaches: + del self._cache[key] + + def stemWord(self, word): + result = self._stem_word(word) + if len(self._cache) > self.maxCacheSize: + self._clear_cache() + return result + + def stemWords(self, words): + result = [self._stem_word(word) for word in words] + if len(self._cache) > self.maxCacheSize: + self._clear_cache() + return result diff --git a/contrib/snowballstemmer/danish_stemmer.py b/contrib/snowballstemmer/danish_stemmer.py new file mode 100644 index 0000000..02721c6 --- /dev/null +++ b/contrib/snowballstemmer/danish_stemmer.py @@ -0,0 +1,364 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class DanishStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"hed", -1, 1), + Among(u"ethed", 0, 1), + Among(u"ered", -1, 1), + Among(u"e", -1, 1), + Among(u"erede", 3, 1), + Among(u"ende", 3, 1), + Among(u"erende", 5, 1), + Among(u"ene", 3, 1), + Among(u"erne", 3, 1), + Among(u"ere", 3, 1), + Among(u"en", -1, 1), + Among(u"heden", 10, 1), + Among(u"eren", 10, 1), + Among(u"er", -1, 1), + Among(u"heder", 13, 1), + Among(u"erer", 13, 1), + Among(u"s", -1, 2), + Among(u"heds", 16, 1), + Among(u"es", 16, 1), + Among(u"endes", 18, 1), + Among(u"erendes", 19, 1), + Among(u"enes", 18, 1), + Among(u"ernes", 18, 1), + Among(u"eres", 18, 1), + Among(u"ens", 16, 1), + Among(u"hedens", 24, 1), + Among(u"erens", 24, 1), + Among(u"ers", 16, 1), + Among(u"ets", 16, 1), + Among(u"erets", 28, 1), + Among(u"et", -1, 1), + Among(u"eret", 30, 1) + ] + + a_1 = [ + Among(u"gd", -1, -1), + Among(u"dt", -1, -1), + Among(u"gt", -1, -1), + Among(u"kt", -1, -1) + ] + + a_2 = [ + Among(u"ig", -1, 1), + Among(u"lig", 0, 1), + Among(u"elig", 1, 1), + Among(u"els", -1, 1), + Among(u"l\u00F8st", -1, 2) + ] + + g_v = [17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 128] + + g_s_ending = [239, 254, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16] + + I_x = 0 + I_p1 = 0 + S_ch = "" + + def copy_from(self, other): + self.I_x = other.I_x + self.I_p1 = other.I_p1 + self.S_ch = other.S_ch + super.copy_from(other) + + + def r_mark_regions(self): + # (, line 29 + self.I_p1 = self.limit; + # test, line 33 + v_1 = self.cursor + # (, line 33 + # hop, line 33 + c = self.cursor + 3 + if 0 > c or c > self.limit: + return False + self.cursor = c + # setmark x, line 33 + self.I_x = self.cursor + self.cursor = v_1 + # goto, line 34 + try: + while True: + v_2 = self.cursor + try: + if not self.in_grouping(DanishStemmer.g_v, 97, 248): + raise lab1() + self.cursor = v_2 + raise lab0() + except lab1: pass + self.cursor = v_2 + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab0: pass + # gopast, line 34 + try: + while True: + try: + if not self.out_grouping(DanishStemmer.g_v, 97, 248): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab2: pass + # setmark p1, line 34 + self.I_p1 = self.cursor + # try, line 35 + try: + # (, line 35 + if not (self.I_p1 < self.I_x): + raise lab4() + self.I_p1 = self.I_x; + except lab4: pass + return True + + def r_main_suffix(self): + # (, line 40 + # setlimit, line 41 + v_1 = self.limit - self.cursor + # tomark, line 41 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 41 + # [, line 41 + self.ket = self.cursor + # substring, line 41 + among_var = self.find_among_b(DanishStemmer.a_0, 32) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 41 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 48 + # delete, line 48 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 50 + if not self.in_grouping_b(DanishStemmer.g_s_ending, 97, 229): + return False + # delete, line 50 + if not self.slice_del(): + return False + + return True + + def r_consonant_pair(self): + # (, line 54 + # test, line 55 + v_1 = self.limit - self.cursor + # (, line 55 + # setlimit, line 56 + v_2 = self.limit - self.cursor + # tomark, line 56 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_3 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_2 + # (, line 56 + # [, line 56 + self.ket = self.cursor + # substring, line 56 + if self.find_among_b(DanishStemmer.a_1, 4) == 0: + self.limit_backward = v_3 + return False + # ], line 56 + self.bra = self.cursor + self.limit_backward = v_3 + self.cursor = self.limit - v_1 + # next, line 62 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # ], line 62 + self.bra = self.cursor + # delete, line 62 + if not self.slice_del(): + return False + + return True + + def r_other_suffix(self): + # (, line 65 + # do, line 66 + v_1 = self.limit - self.cursor + try: + # (, line 66 + # [, line 66 + self.ket = self.cursor + # literal, line 66 + if not self.eq_s_b(2, u"st"): + raise lab0() + # ], line 66 + self.bra = self.cursor + # literal, line 66 + if not self.eq_s_b(2, u"ig"): + raise lab0() + # delete, line 66 + if not self.slice_del(): + return False + + except lab0: pass + self.cursor = self.limit - v_1 + # setlimit, line 67 + v_2 = self.limit - self.cursor + # tomark, line 67 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_3 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_2 + # (, line 67 + # [, line 67 + self.ket = self.cursor + # substring, line 67 + among_var = self.find_among_b(DanishStemmer.a_2, 5) + if among_var == 0: + self.limit_backward = v_3 + return False + # ], line 67 + self.bra = self.cursor + self.limit_backward = v_3 + if among_var == 0: + return False + elif among_var == 1: + # (, line 70 + # delete, line 70 + if not self.slice_del(): + return False + + # do, line 70 + v_4 = self.limit - self.cursor + try: + # call consonant_pair, line 70 + if not self.r_consonant_pair(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_4 + elif among_var == 2: + # (, line 72 + # <-, line 72 + if not self.slice_from(u"l\u00F8s"): + return False + return True + + def r_undouble(self): + # (, line 75 + # setlimit, line 76 + v_1 = self.limit - self.cursor + # tomark, line 76 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 76 + # [, line 76 + self.ket = self.cursor + if not self.out_grouping_b(DanishStemmer.g_v, 97, 248): + self.limit_backward = v_2 + return False + # ], line 76 + self.bra = self.cursor + # -> ch, line 76 + self.S_ch = self.slice_to(self.S_ch) + if self.S_ch == '': + return False + self.limit_backward = v_2 + # name ch, line 77 + if not self.eq_v_b(self.S_ch): + return False + # delete, line 78 + if not self.slice_del(): + return False + + return True + + def _stem(self): + # (, line 82 + # do, line 84 + v_1 = self.cursor + try: + # call mark_regions, line 84 + if not self.r_mark_regions(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # backwards, line 85 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 85 + # do, line 86 + v_2 = self.limit - self.cursor + try: + # call main_suffix, line 86 + if not self.r_main_suffix(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 87 + v_3 = self.limit - self.cursor + try: + # call consonant_pair, line 87 + if not self.r_consonant_pair(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 88 + v_4 = self.limit - self.cursor + try: + # call other_suffix, line 88 + if not self.r_other_suffix(): + raise lab3() + except lab3: pass + self.cursor = self.limit - v_4 + # do, line 89 + v_5 = self.limit - self.cursor + try: + # call undouble, line 89 + if not self.r_undouble(): + raise lab4() + except lab4: pass + self.cursor = self.limit - v_5 + self.cursor = self.limit_backward + return True + + def equals(self, o): + return isinstance(o, DanishStemmer) + + def hashCode(self): + return hash("DanishStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass diff --git a/contrib/snowballstemmer/dutch_stemmer.py b/contrib/snowballstemmer/dutch_stemmer.py new file mode 100644 index 0000000..a6f1e7d --- /dev/null +++ b/contrib/snowballstemmer/dutch_stemmer.py @@ -0,0 +1,699 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class DutchStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"", -1, 6), + Among(u"\u00E1", 0, 1), + Among(u"\u00E4", 0, 1), + Among(u"\u00E9", 0, 2), + Among(u"\u00EB", 0, 2), + Among(u"\u00ED", 0, 3), + Among(u"\u00EF", 0, 3), + Among(u"\u00F3", 0, 4), + Among(u"\u00F6", 0, 4), + Among(u"\u00FA", 0, 5), + Among(u"\u00FC", 0, 5) + ] + + a_1 = [ + Among(u"", -1, 3), + Among(u"I", 0, 2), + Among(u"Y", 0, 1) + ] + + a_2 = [ + Among(u"dd", -1, -1), + Among(u"kk", -1, -1), + Among(u"tt", -1, -1) + ] + + a_3 = [ + Among(u"ene", -1, 2), + Among(u"se", -1, 3), + Among(u"en", -1, 2), + Among(u"heden", 2, 1), + Among(u"s", -1, 3) + ] + + a_4 = [ + Among(u"end", -1, 1), + Among(u"ig", -1, 2), + Among(u"ing", -1, 1), + Among(u"lijk", -1, 3), + Among(u"baar", -1, 4), + Among(u"bar", -1, 5) + ] + + a_5 = [ + Among(u"aa", -1, -1), + Among(u"ee", -1, -1), + Among(u"oo", -1, -1), + Among(u"uu", -1, -1) + ] + + g_v = [17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128] + + g_v_I = [1, 0, 0, 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128] + + g_v_j = [17, 67, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128] + + I_p2 = 0 + I_p1 = 0 + B_e_found = False + + def copy_from(self, other): + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + self.B_e_found = other.B_e_found + super.copy_from(other) + + + def r_prelude(self): + # (, line 41 + # test, line 42 + v_1 = self.cursor + # repeat, line 42 + try: + while True: + try: + v_2 = self.cursor + try: + # (, line 42 + # [, line 43 + self.bra = self.cursor + # substring, line 43 + among_var = self.find_among(DutchStemmer.a_0, 11) + if among_var == 0: + raise lab2() + # ], line 43 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 45 + # <-, line 45 + if not self.slice_from(u"a"): + return False + elif among_var == 2: + # (, line 47 + # <-, line 47 + if not self.slice_from(u"e"): + return False + elif among_var == 3: + # (, line 49 + # <-, line 49 + if not self.slice_from(u"i"): + return False + elif among_var == 4: + # (, line 51 + # <-, line 51 + if not self.slice_from(u"o"): + return False + elif among_var == 5: + # (, line 53 + # <-, line 53 + if not self.slice_from(u"u"): + return False + elif among_var == 6: + # (, line 54 + # next, line 54 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_2 + raise lab0() + except lab1: pass + except lab0: pass + self.cursor = v_1 + # try, line 57 + v_3 = self.cursor + try: + # (, line 57 + # [, line 57 + self.bra = self.cursor + # literal, line 57 + if not self.eq_s(1, u"y"): + self.cursor = v_3 + raise lab3() + # ], line 57 + self.ket = self.cursor + # <-, line 57 + if not self.slice_from(u"Y"): + return False + except lab3: pass + # repeat, line 58 + try: + while True: + try: + v_4 = self.cursor + try: + # goto, line 58 + try: + while True: + v_5 = self.cursor + try: + # (, line 58 + if not self.in_grouping(DutchStemmer.g_v, 97, 232): + raise lab8() + # [, line 59 + self.bra = self.cursor + # or, line 59 + try: + v_6 = self.cursor + try: + # (, line 59 + # literal, line 59 + if not self.eq_s(1, u"i"): + raise lab10() + # ], line 59 + self.ket = self.cursor + if not self.in_grouping(DutchStemmer.g_v, 97, 232): + raise lab10() + # <-, line 59 + if not self.slice_from(u"I"): + return False + raise lab9() + except lab10: pass + self.cursor = v_6 + # (, line 60 + # literal, line 60 + if not self.eq_s(1, u"y"): + raise lab8() + # ], line 60 + self.ket = self.cursor + # <-, line 60 + if not self.slice_from(u"Y"): + return False + except lab9: pass + self.cursor = v_5 + raise lab7() + except lab8: pass + self.cursor = v_5 + if self.cursor >= self.limit: + raise lab6() + self.cursor += 1 + except lab7: pass + raise lab5() + except lab6: pass + self.cursor = v_4 + raise lab4() + except lab5: pass + except lab4: pass + return True + + def r_mark_regions(self): + # (, line 64 + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # gopast, line 69 + try: + while True: + try: + if not self.in_grouping(DutchStemmer.g_v, 97, 232): + raise lab1() + raise lab0() + except lab1: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab0: pass + # gopast, line 69 + try: + while True: + try: + if not self.out_grouping(DutchStemmer.g_v, 97, 232): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab2: pass + # setmark p1, line 69 + self.I_p1 = self.cursor + # try, line 70 + try: + # (, line 70 + if not self.I_p1 < 3: + raise lab4() + self.I_p1 = 3; + except lab4: pass + # gopast, line 71 + try: + while True: + try: + if not self.in_grouping(DutchStemmer.g_v, 97, 232): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab5: pass + # gopast, line 71 + try: + while True: + try: + if not self.out_grouping(DutchStemmer.g_v, 97, 232): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab7: pass + # setmark p2, line 71 + self.I_p2 = self.cursor + return True + + def r_postlude(self): + # repeat, line 75 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 75 + # [, line 77 + self.bra = self.cursor + # substring, line 77 + among_var = self.find_among(DutchStemmer.a_1, 3) + if among_var == 0: + raise lab2() + # ], line 77 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 78 + # <-, line 78 + if not self.slice_from(u"y"): + return False + elif among_var == 2: + # (, line 79 + # <-, line 79 + if not self.slice_from(u"i"): + return False + elif among_var == 3: + # (, line 80 + # next, line 80 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_undouble(self): + # (, line 90 + # test, line 91 + v_1 = self.limit - self.cursor + # among, line 91 + if self.find_among_b(DutchStemmer.a_2, 3) == 0: + return False + self.cursor = self.limit - v_1 + # [, line 91 + self.ket = self.cursor + # next, line 91 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # ], line 91 + self.bra = self.cursor + # delete, line 91 + if not self.slice_del(): + return False + + return True + + def r_e_ending(self): + # (, line 94 + # unset e_found, line 95 + self.B_e_found = False + # [, line 96 + self.ket = self.cursor + # literal, line 96 + if not self.eq_s_b(1, u"e"): + return False + # ], line 96 + self.bra = self.cursor + # call R1, line 96 + if not self.r_R1(): + return False + # test, line 96 + v_1 = self.limit - self.cursor + if not self.out_grouping_b(DutchStemmer.g_v, 97, 232): + return False + self.cursor = self.limit - v_1 + # delete, line 96 + if not self.slice_del(): + return False + + # set e_found, line 97 + self.B_e_found = True + # call undouble, line 98 + if not self.r_undouble(): + return False + return True + + def r_en_ending(self): + # (, line 101 + # call R1, line 102 + if not self.r_R1(): + return False + # and, line 102 + v_1 = self.limit - self.cursor + if not self.out_grouping_b(DutchStemmer.g_v, 97, 232): + return False + self.cursor = self.limit - v_1 + # not, line 102 + v_2 = self.limit - self.cursor + try: + # literal, line 102 + if not self.eq_s_b(3, u"gem"): + raise lab0() + return False + except lab0: pass + self.cursor = self.limit - v_2 + # delete, line 102 + if not self.slice_del(): + return False + + # call undouble, line 103 + if not self.r_undouble(): + return False + return True + + def r_standard_suffix(self): + # (, line 106 + # do, line 107 + v_1 = self.limit - self.cursor + try: + # (, line 107 + # [, line 108 + self.ket = self.cursor + # substring, line 108 + among_var = self.find_among_b(DutchStemmer.a_3, 5) + if among_var == 0: + raise lab0() + # ], line 108 + self.bra = self.cursor + if among_var == 0: + raise lab0() + elif among_var == 1: + # (, line 110 + # call R1, line 110 + if not self.r_R1(): + raise lab0() + # <-, line 110 + if not self.slice_from(u"heid"): + return False + elif among_var == 2: + # (, line 113 + # call en_ending, line 113 + if not self.r_en_ending(): + raise lab0() + elif among_var == 3: + # (, line 116 + # call R1, line 116 + if not self.r_R1(): + raise lab0() + if not self.out_grouping_b(DutchStemmer.g_v_j, 97, 232): + raise lab0() + # delete, line 116 + if not self.slice_del(): + return False + + except lab0: pass + self.cursor = self.limit - v_1 + # do, line 120 + v_2 = self.limit - self.cursor + try: + # call e_ending, line 120 + if not self.r_e_ending(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 122 + v_3 = self.limit - self.cursor + try: + # (, line 122 + # [, line 122 + self.ket = self.cursor + # literal, line 122 + if not self.eq_s_b(4, u"heid"): + raise lab2() + # ], line 122 + self.bra = self.cursor + # call R2, line 122 + if not self.r_R2(): + raise lab2() + # not, line 122 + v_4 = self.limit - self.cursor + try: + # literal, line 122 + if not self.eq_s_b(1, u"c"): + raise lab3() + raise lab2() + except lab3: pass + self.cursor = self.limit - v_4 + # delete, line 122 + if not self.slice_del(): + return False + + # [, line 123 + self.ket = self.cursor + # literal, line 123 + if not self.eq_s_b(2, u"en"): + raise lab2() + # ], line 123 + self.bra = self.cursor + # call en_ending, line 123 + if not self.r_en_ending(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 126 + v_5 = self.limit - self.cursor + try: + # (, line 126 + # [, line 127 + self.ket = self.cursor + # substring, line 127 + among_var = self.find_among_b(DutchStemmer.a_4, 6) + if among_var == 0: + raise lab4() + # ], line 127 + self.bra = self.cursor + if among_var == 0: + raise lab4() + elif among_var == 1: + # (, line 129 + # call R2, line 129 + if not self.r_R2(): + raise lab4() + # delete, line 129 + if not self.slice_del(): + return False + + # or, line 130 + try: + v_6 = self.limit - self.cursor + try: + # (, line 130 + # [, line 130 + self.ket = self.cursor + # literal, line 130 + if not self.eq_s_b(2, u"ig"): + raise lab6() + # ], line 130 + self.bra = self.cursor + # call R2, line 130 + if not self.r_R2(): + raise lab6() + # not, line 130 + v_7 = self.limit - self.cursor + try: + # literal, line 130 + if not self.eq_s_b(1, u"e"): + raise lab7() + raise lab6() + except lab7: pass + self.cursor = self.limit - v_7 + # delete, line 130 + if not self.slice_del(): + return False + + raise lab5() + except lab6: pass + self.cursor = self.limit - v_6 + # call undouble, line 130 + if not self.r_undouble(): + raise lab4() + except lab5: pass + elif among_var == 2: + # (, line 133 + # call R2, line 133 + if not self.r_R2(): + raise lab4() + # not, line 133 + v_8 = self.limit - self.cursor + try: + # literal, line 133 + if not self.eq_s_b(1, u"e"): + raise lab8() + raise lab4() + except lab8: pass + self.cursor = self.limit - v_8 + # delete, line 133 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 136 + # call R2, line 136 + if not self.r_R2(): + raise lab4() + # delete, line 136 + if not self.slice_del(): + return False + + # call e_ending, line 136 + if not self.r_e_ending(): + raise lab4() + elif among_var == 4: + # (, line 139 + # call R2, line 139 + if not self.r_R2(): + raise lab4() + # delete, line 139 + if not self.slice_del(): + return False + + elif among_var == 5: + # (, line 142 + # call R2, line 142 + if not self.r_R2(): + raise lab4() + # Boolean test e_found, line 142 + if not self.B_e_found: + raise lab4() + # delete, line 142 + if not self.slice_del(): + return False + + except lab4: pass + self.cursor = self.limit - v_5 + # do, line 146 + v_9 = self.limit - self.cursor + try: + # (, line 146 + if not self.out_grouping_b(DutchStemmer.g_v_I, 73, 232): + raise lab9() + # test, line 148 + v_10 = self.limit - self.cursor + # (, line 148 + # among, line 149 + if self.find_among_b(DutchStemmer.a_5, 4) == 0: + raise lab9() + if not self.out_grouping_b(DutchStemmer.g_v, 97, 232): + raise lab9() + self.cursor = self.limit - v_10 + # [, line 152 + self.ket = self.cursor + # next, line 152 + if self.cursor <= self.limit_backward: + raise lab9() + self.cursor -= 1 + # ], line 152 + self.bra = self.cursor + # delete, line 152 + if not self.slice_del(): + return False + + except lab9: pass + self.cursor = self.limit - v_9 + return True + + def _stem(self): + # (, line 157 + # do, line 159 + v_1 = self.cursor + try: + # call prelude, line 159 + if not self.r_prelude(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # do, line 160 + v_2 = self.cursor + try: + # call mark_regions, line 160 + if not self.r_mark_regions(): + raise lab1() + except lab1: pass + self.cursor = v_2 + # backwards, line 161 + self.limit_backward = self.cursor + self.cursor = self.limit + # do, line 162 + v_3 = self.limit - self.cursor + try: + # call standard_suffix, line 162 + if not self.r_standard_suffix(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + self.cursor = self.limit_backward + # do, line 163 + v_4 = self.cursor + try: + # call postlude, line 163 + if not self.r_postlude(): + raise lab3() + except lab3: pass + self.cursor = v_4 + return True + + def equals(self, o): + return isinstance(o, DutchStemmer) + + def hashCode(self): + return hash("DutchStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass diff --git a/contrib/snowballstemmer/english_stemmer.py b/contrib/snowballstemmer/english_stemmer.py new file mode 100644 index 0000000..dccbc4b --- /dev/null +++ b/contrib/snowballstemmer/english_stemmer.py @@ -0,0 +1,1115 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class EnglishStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"arsen", -1, -1), + Among(u"commun", -1, -1), + Among(u"gener", -1, -1) + ] + + a_1 = [ + Among(u"'", -1, 1), + Among(u"'s'", 0, 1), + Among(u"'s", -1, 1) + ] + + a_2 = [ + Among(u"ied", -1, 2), + Among(u"s", -1, 3), + Among(u"ies", 1, 2), + Among(u"sses", 1, 1), + Among(u"ss", 1, -1), + Among(u"us", 1, -1) + ] + + a_3 = [ + Among(u"", -1, 3), + Among(u"bb", 0, 2), + Among(u"dd", 0, 2), + Among(u"ff", 0, 2), + Among(u"gg", 0, 2), + Among(u"bl", 0, 1), + Among(u"mm", 0, 2), + Among(u"nn", 0, 2), + Among(u"pp", 0, 2), + Among(u"rr", 0, 2), + Among(u"at", 0, 1), + Among(u"tt", 0, 2), + Among(u"iz", 0, 1) + ] + + a_4 = [ + Among(u"ed", -1, 2), + Among(u"eed", 0, 1), + Among(u"ing", -1, 2), + Among(u"edly", -1, 2), + Among(u"eedly", 3, 1), + Among(u"ingly", -1, 2) + ] + + a_5 = [ + Among(u"anci", -1, 3), + Among(u"enci", -1, 2), + Among(u"ogi", -1, 13), + Among(u"li", -1, 16), + Among(u"bli", 3, 12), + Among(u"abli", 4, 4), + Among(u"alli", 3, 8), + Among(u"fulli", 3, 14), + Among(u"lessli", 3, 15), + Among(u"ousli", 3, 10), + Among(u"entli", 3, 5), + Among(u"aliti", -1, 8), + Among(u"biliti", -1, 12), + Among(u"iviti", -1, 11), + Among(u"tional", -1, 1), + Among(u"ational", 14, 7), + Among(u"alism", -1, 8), + Among(u"ation", -1, 7), + Among(u"ization", 17, 6), + Among(u"izer", -1, 6), + Among(u"ator", -1, 7), + Among(u"iveness", -1, 11), + Among(u"fulness", -1, 9), + Among(u"ousness", -1, 10) + ] + + a_6 = [ + Among(u"icate", -1, 4), + Among(u"ative", -1, 6), + Among(u"alize", -1, 3), + Among(u"iciti", -1, 4), + Among(u"ical", -1, 4), + Among(u"tional", -1, 1), + Among(u"ational", 5, 2), + Among(u"ful", -1, 5), + Among(u"ness", -1, 5) + ] + + a_7 = [ + Among(u"ic", -1, 1), + Among(u"ance", -1, 1), + Among(u"ence", -1, 1), + Among(u"able", -1, 1), + Among(u"ible", -1, 1), + Among(u"ate", -1, 1), + Among(u"ive", -1, 1), + Among(u"ize", -1, 1), + Among(u"iti", -1, 1), + Among(u"al", -1, 1), + Among(u"ism", -1, 1), + Among(u"ion", -1, 2), + Among(u"er", -1, 1), + Among(u"ous", -1, 1), + Among(u"ant", -1, 1), + Among(u"ent", -1, 1), + Among(u"ment", 15, 1), + Among(u"ement", 16, 1) + ] + + a_8 = [ + Among(u"e", -1, 1), + Among(u"l", -1, 2) + ] + + a_9 = [ + Among(u"succeed", -1, -1), + Among(u"proceed", -1, -1), + Among(u"exceed", -1, -1), + Among(u"canning", -1, -1), + Among(u"inning", -1, -1), + Among(u"earring", -1, -1), + Among(u"herring", -1, -1), + Among(u"outing", -1, -1) + ] + + a_10 = [ + Among(u"andes", -1, -1), + Among(u"atlas", -1, -1), + Among(u"bias", -1, -1), + Among(u"cosmos", -1, -1), + Among(u"dying", -1, 3), + Among(u"early", -1, 9), + Among(u"gently", -1, 7), + Among(u"howe", -1, -1), + Among(u"idly", -1, 6), + Among(u"lying", -1, 4), + Among(u"news", -1, -1), + Among(u"only", -1, 10), + Among(u"singly", -1, 11), + Among(u"skies", -1, 2), + Among(u"skis", -1, 1), + Among(u"sky", -1, -1), + Among(u"tying", -1, 5), + Among(u"ugly", -1, 8) + ] + + g_v = [17, 65, 16, 1] + + g_v_WXY = [1, 17, 65, 208, 1] + + g_valid_LI = [55, 141, 2] + + B_Y_found = False + I_p2 = 0 + I_p1 = 0 + + def copy_from(self, other): + self.B_Y_found = other.B_Y_found + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + super.copy_from(other) + + + def r_prelude(self): + # (, line 25 + # unset Y_found, line 26 + self.B_Y_found = False + # do, line 27 + v_1 = self.cursor + try: + # (, line 27 + # [, line 27 + self.bra = self.cursor + # literal, line 27 + if not self.eq_s(1, u"'"): + raise lab0() + # ], line 27 + self.ket = self.cursor + # delete, line 27 + if not self.slice_del(): + return False + + except lab0: pass + self.cursor = v_1 + # do, line 28 + v_2 = self.cursor + try: + # (, line 28 + # [, line 28 + self.bra = self.cursor + # literal, line 28 + if not self.eq_s(1, u"y"): + raise lab1() + # ], line 28 + self.ket = self.cursor + # <-, line 28 + if not self.slice_from(u"Y"): + return False + # set Y_found, line 28 + self.B_Y_found = True + except lab1: pass + self.cursor = v_2 + # do, line 29 + v_3 = self.cursor + try: + # repeat, line 29 + try: + while True: + try: + v_4 = self.cursor + try: + # (, line 29 + # goto, line 29 + try: + while True: + v_5 = self.cursor + try: + # (, line 29 + if not self.in_grouping(EnglishStemmer.g_v, 97, 121): + raise lab7() + # [, line 29 + self.bra = self.cursor + # literal, line 29 + if not self.eq_s(1, u"y"): + raise lab7() + # ], line 29 + self.ket = self.cursor + self.cursor = v_5 + raise lab6() + except lab7: pass + self.cursor = v_5 + if self.cursor >= self.limit: + raise lab5() + self.cursor += 1 + except lab6: pass + # <-, line 29 + if not self.slice_from(u"Y"): + return False + # set Y_found, line 29 + self.B_Y_found = True + raise lab4() + except lab5: pass + self.cursor = v_4 + raise lab3() + except lab4: pass + except lab3: pass + except lab2: pass + self.cursor = v_3 + return True + + def r_mark_regions(self): + # (, line 32 + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # do, line 35 + v_1 = self.cursor + try: + # (, line 35 + # or, line 41 + try: + v_2 = self.cursor + try: + # among, line 36 + if self.find_among(EnglishStemmer.a_0, 3) == 0: + raise lab2() + raise lab1() + except lab2: pass + self.cursor = v_2 + # (, line 41 + # gopast, line 41 + try: + while True: + try: + if not self.in_grouping(EnglishStemmer.g_v, 97, 121): + raise lab4() + raise lab3() + except lab4: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab3: pass + # gopast, line 41 + try: + while True: + try: + if not self.out_grouping(EnglishStemmer.g_v, 97, 121): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab5: pass + except lab1: pass + # setmark p1, line 42 + self.I_p1 = self.cursor + # gopast, line 43 + try: + while True: + try: + if not self.in_grouping(EnglishStemmer.g_v, 97, 121): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab7: pass + # gopast, line 43 + try: + while True: + try: + if not self.out_grouping(EnglishStemmer.g_v, 97, 121): + raise lab10() + raise lab9() + except lab10: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab9: pass + # setmark p2, line 43 + self.I_p2 = self.cursor + except lab0: pass + self.cursor = v_1 + return True + + def r_shortv(self): + # (, line 49 + # or, line 51 + try: + v_1 = self.limit - self.cursor + try: + # (, line 50 + if not self.out_grouping_b(EnglishStemmer.g_v_WXY, 89, 121): + raise lab1() + if not self.in_grouping_b(EnglishStemmer.g_v, 97, 121): + raise lab1() + if not self.out_grouping_b(EnglishStemmer.g_v, 97, 121): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 52 + if not self.out_grouping_b(EnglishStemmer.g_v, 97, 121): + return False + if not self.in_grouping_b(EnglishStemmer.g_v, 97, 121): + return False + # atlimit, line 52 + if self.cursor > self.limit_backward: + return False + except lab0: pass + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_Step_1a(self): + # (, line 58 + # try, line 59 + v_1 = self.limit - self.cursor + try: + # (, line 59 + # [, line 60 + self.ket = self.cursor + # substring, line 60 + among_var = self.find_among_b(EnglishStemmer.a_1, 3) + if among_var == 0: + self.cursor = self.limit - v_1 + raise lab0() + # ], line 60 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_1 + raise lab0() + elif among_var == 1: + # (, line 62 + # delete, line 62 + if not self.slice_del(): + return False + + except lab0: pass + # [, line 65 + self.ket = self.cursor + # substring, line 65 + among_var = self.find_among_b(EnglishStemmer.a_2, 6) + if among_var == 0: + return False + # ], line 65 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 66 + # <-, line 66 + if not self.slice_from(u"ss"): + return False + elif among_var == 2: + # (, line 68 + # or, line 68 + try: + v_2 = self.limit - self.cursor + try: + # (, line 68 + # hop, line 68 + c = self.cursor - 2 + if self.limit_backward > c or c > self.limit: + raise lab2() + self.cursor = c + # <-, line 68 + if not self.slice_from(u"i"): + return False + raise lab1() + except lab2: pass + self.cursor = self.limit - v_2 + # <-, line 68 + if not self.slice_from(u"ie"): + return False + except lab1: pass + elif among_var == 3: + # (, line 69 + # next, line 69 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # gopast, line 69 + try: + while True: + try: + if not self.in_grouping_b(EnglishStemmer.g_v, 97, 121): + raise lab4() + raise lab3() + except lab4: pass + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab3: pass + # delete, line 69 + if not self.slice_del(): + return False + + return True + + def r_Step_1b(self): + # (, line 74 + # [, line 75 + self.ket = self.cursor + # substring, line 75 + among_var = self.find_among_b(EnglishStemmer.a_4, 6) + if among_var == 0: + return False + # ], line 75 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 77 + # call R1, line 77 + if not self.r_R1(): + return False + # <-, line 77 + if not self.slice_from(u"ee"): + return False + elif among_var == 2: + # (, line 79 + # test, line 80 + v_1 = self.limit - self.cursor + # gopast, line 80 + try: + while True: + try: + if not self.in_grouping_b(EnglishStemmer.g_v, 97, 121): + raise lab1() + raise lab0() + except lab1: pass + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab0: pass + self.cursor = self.limit - v_1 + # delete, line 80 + if not self.slice_del(): + return False + + # test, line 81 + v_3 = self.limit - self.cursor + # substring, line 81 + among_var = self.find_among_b(EnglishStemmer.a_3, 13) + if among_var == 0: + return False + self.cursor = self.limit - v_3 + if among_var == 0: + return False + elif among_var == 1: + # (, line 83 + # <+, line 83 + c = self.cursor + self.insert(self.cursor, self.cursor, u"e") + self.cursor = c + elif among_var == 2: + # (, line 86 + # [, line 86 + self.ket = self.cursor + # next, line 86 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # ], line 86 + self.bra = self.cursor + # delete, line 86 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 87 + # atmark, line 87 + if self.cursor != self.I_p1: + return False + # test, line 87 + v_4 = self.limit - self.cursor + # call shortv, line 87 + if not self.r_shortv(): + return False + self.cursor = self.limit - v_4 + # <+, line 87 + c = self.cursor + self.insert(self.cursor, self.cursor, u"e") + self.cursor = c + return True + + def r_Step_1c(self): + # (, line 93 + # [, line 94 + self.ket = self.cursor + # or, line 94 + try: + v_1 = self.limit - self.cursor + try: + # literal, line 94 + if not self.eq_s_b(1, u"y"): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # literal, line 94 + if not self.eq_s_b(1, u"Y"): + return False + except lab0: pass + # ], line 94 + self.bra = self.cursor + if not self.out_grouping_b(EnglishStemmer.g_v, 97, 121): + return False + # not, line 95 + v_2 = self.limit - self.cursor + try: + # atlimit, line 95 + if self.cursor > self.limit_backward: + raise lab2() + return False + except lab2: pass + self.cursor = self.limit - v_2 + # <-, line 96 + if not self.slice_from(u"i"): + return False + return True + + def r_Step_2(self): + # (, line 99 + # [, line 100 + self.ket = self.cursor + # substring, line 100 + among_var = self.find_among_b(EnglishStemmer.a_5, 24) + if among_var == 0: + return False + # ], line 100 + self.bra = self.cursor + # call R1, line 100 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 101 + # <-, line 101 + if not self.slice_from(u"tion"): + return False + elif among_var == 2: + # (, line 102 + # <-, line 102 + if not self.slice_from(u"ence"): + return False + elif among_var == 3: + # (, line 103 + # <-, line 103 + if not self.slice_from(u"ance"): + return False + elif among_var == 4: + # (, line 104 + # <-, line 104 + if not self.slice_from(u"able"): + return False + elif among_var == 5: + # (, line 105 + # <-, line 105 + if not self.slice_from(u"ent"): + return False + elif among_var == 6: + # (, line 107 + # <-, line 107 + if not self.slice_from(u"ize"): + return False + elif among_var == 7: + # (, line 109 + # <-, line 109 + if not self.slice_from(u"ate"): + return False + elif among_var == 8: + # (, line 111 + # <-, line 111 + if not self.slice_from(u"al"): + return False + elif among_var == 9: + # (, line 112 + # <-, line 112 + if not self.slice_from(u"ful"): + return False + elif among_var == 10: + # (, line 114 + # <-, line 114 + if not self.slice_from(u"ous"): + return False + elif among_var == 11: + # (, line 116 + # <-, line 116 + if not self.slice_from(u"ive"): + return False + elif among_var == 12: + # (, line 118 + # <-, line 118 + if not self.slice_from(u"ble"): + return False + elif among_var == 13: + # (, line 119 + # literal, line 119 + if not self.eq_s_b(1, u"l"): + return False + # <-, line 119 + if not self.slice_from(u"og"): + return False + elif among_var == 14: + # (, line 120 + # <-, line 120 + if not self.slice_from(u"ful"): + return False + elif among_var == 15: + # (, line 121 + # <-, line 121 + if not self.slice_from(u"less"): + return False + elif among_var == 16: + # (, line 122 + if not self.in_grouping_b(EnglishStemmer.g_valid_LI, 99, 116): + return False + # delete, line 122 + if not self.slice_del(): + return False + + return True + + def r_Step_3(self): + # (, line 126 + # [, line 127 + self.ket = self.cursor + # substring, line 127 + among_var = self.find_among_b(EnglishStemmer.a_6, 9) + if among_var == 0: + return False + # ], line 127 + self.bra = self.cursor + # call R1, line 127 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 128 + # <-, line 128 + if not self.slice_from(u"tion"): + return False + elif among_var == 2: + # (, line 129 + # <-, line 129 + if not self.slice_from(u"ate"): + return False + elif among_var == 3: + # (, line 130 + # <-, line 130 + if not self.slice_from(u"al"): + return False + elif among_var == 4: + # (, line 132 + # <-, line 132 + if not self.slice_from(u"ic"): + return False + elif among_var == 5: + # (, line 134 + # delete, line 134 + if not self.slice_del(): + return False + + elif among_var == 6: + # (, line 136 + # call R2, line 136 + if not self.r_R2(): + return False + # delete, line 136 + if not self.slice_del(): + return False + + return True + + def r_Step_4(self): + # (, line 140 + # [, line 141 + self.ket = self.cursor + # substring, line 141 + among_var = self.find_among_b(EnglishStemmer.a_7, 18) + if among_var == 0: + return False + # ], line 141 + self.bra = self.cursor + # call R2, line 141 + if not self.r_R2(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 144 + # delete, line 144 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 145 + # or, line 145 + try: + v_1 = self.limit - self.cursor + try: + # literal, line 145 + if not self.eq_s_b(1, u"s"): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # literal, line 145 + if not self.eq_s_b(1, u"t"): + return False + except lab0: pass + # delete, line 145 + if not self.slice_del(): + return False + + return True + + def r_Step_5(self): + # (, line 149 + # [, line 150 + self.ket = self.cursor + # substring, line 150 + among_var = self.find_among_b(EnglishStemmer.a_8, 2) + if among_var == 0: + return False + # ], line 150 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 151 + # or, line 151 + try: + v_1 = self.limit - self.cursor + try: + # call R2, line 151 + if not self.r_R2(): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 151 + # call R1, line 151 + if not self.r_R1(): + return False + # not, line 151 + v_2 = self.limit - self.cursor + try: + # call shortv, line 151 + if not self.r_shortv(): + raise lab2() + return False + except lab2: pass + self.cursor = self.limit - v_2 + except lab0: pass + # delete, line 151 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 152 + # call R2, line 152 + if not self.r_R2(): + return False + # literal, line 152 + if not self.eq_s_b(1, u"l"): + return False + # delete, line 152 + if not self.slice_del(): + return False + + return True + + def r_exception2(self): + # (, line 156 + # [, line 158 + self.ket = self.cursor + # substring, line 158 + if self.find_among_b(EnglishStemmer.a_9, 8) == 0: + return False + # ], line 158 + self.bra = self.cursor + # atlimit, line 158 + if self.cursor > self.limit_backward: + return False + return True + + def r_exception1(self): + # (, line 168 + # [, line 170 + self.bra = self.cursor + # substring, line 170 + among_var = self.find_among(EnglishStemmer.a_10, 18) + if among_var == 0: + return False + # ], line 170 + self.ket = self.cursor + # atlimit, line 170 + if self.cursor < self.limit: + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 174 + # <-, line 174 + if not self.slice_from(u"ski"): + return False + elif among_var == 2: + # (, line 175 + # <-, line 175 + if not self.slice_from(u"sky"): + return False + elif among_var == 3: + # (, line 176 + # <-, line 176 + if not self.slice_from(u"die"): + return False + elif among_var == 4: + # (, line 177 + # <-, line 177 + if not self.slice_from(u"lie"): + return False + elif among_var == 5: + # (, line 178 + # <-, line 178 + if not self.slice_from(u"tie"): + return False + elif among_var == 6: + # (, line 182 + # <-, line 182 + if not self.slice_from(u"idl"): + return False + elif among_var == 7: + # (, line 183 + # <-, line 183 + if not self.slice_from(u"gentl"): + return False + elif among_var == 8: + # (, line 184 + # <-, line 184 + if not self.slice_from(u"ugli"): + return False + elif among_var == 9: + # (, line 185 + # <-, line 185 + if not self.slice_from(u"earli"): + return False + elif among_var == 10: + # (, line 186 + # <-, line 186 + if not self.slice_from(u"onli"): + return False + elif among_var == 11: + # (, line 187 + # <-, line 187 + if not self.slice_from(u"singl"): + return False + return True + + def r_postlude(self): + # (, line 203 + # Boolean test Y_found, line 203 + if not self.B_Y_found: + return False + # repeat, line 203 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 203 + # goto, line 203 + try: + while True: + v_2 = self.cursor + try: + # (, line 203 + # [, line 203 + self.bra = self.cursor + # literal, line 203 + if not self.eq_s(1, u"Y"): + raise lab4() + # ], line 203 + self.ket = self.cursor + self.cursor = v_2 + raise lab3() + except lab4: pass + self.cursor = v_2 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab3: pass + # <-, line 203 + if not self.slice_from(u"y"): + return False + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def _stem(self): + # (, line 205 + # or, line 207 + try: + v_1 = self.cursor + try: + # call exception1, line 207 + if not self.r_exception1(): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = v_1 + try: + # not, line 208 + v_2 = self.cursor + try: + # hop, line 208 + c = self.cursor + 3 + if 0 > c or c > self.limit: + raise lab3() + self.cursor = c + raise lab2() + except lab3: pass + self.cursor = v_2 + raise lab0() + except lab2: pass + self.cursor = v_1 + # (, line 208 + # do, line 209 + v_3 = self.cursor + try: + # call prelude, line 209 + if not self.r_prelude(): + raise lab4() + except lab4: pass + self.cursor = v_3 + # do, line 210 + v_4 = self.cursor + try: + # call mark_regions, line 210 + if not self.r_mark_regions(): + raise lab5() + except lab5: pass + self.cursor = v_4 + # backwards, line 211 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 211 + # do, line 213 + v_5 = self.limit - self.cursor + try: + # call Step_1a, line 213 + if not self.r_Step_1a(): + raise lab6() + except lab6: pass + self.cursor = self.limit - v_5 + # or, line 215 + try: + v_6 = self.limit - self.cursor + try: + # call exception2, line 215 + if not self.r_exception2(): + raise lab8() + raise lab7() + except lab8: pass + self.cursor = self.limit - v_6 + # (, line 215 + # do, line 217 + v_7 = self.limit - self.cursor + try: + # call Step_1b, line 217 + if not self.r_Step_1b(): + raise lab9() + except lab9: pass + self.cursor = self.limit - v_7 + # do, line 218 + v_8 = self.limit - self.cursor + try: + # call Step_1c, line 218 + if not self.r_Step_1c(): + raise lab10() + except lab10: pass + self.cursor = self.limit - v_8 + # do, line 220 + v_9 = self.limit - self.cursor + try: + # call Step_2, line 220 + if not self.r_Step_2(): + raise lab11() + except lab11: pass + self.cursor = self.limit - v_9 + # do, line 221 + v_10 = self.limit - self.cursor + try: + # call Step_3, line 221 + if not self.r_Step_3(): + raise lab12() + except lab12: pass + self.cursor = self.limit - v_10 + # do, line 222 + v_11 = self.limit - self.cursor + try: + # call Step_4, line 222 + if not self.r_Step_4(): + raise lab13() + except lab13: pass + self.cursor = self.limit - v_11 + # do, line 224 + v_12 = self.limit - self.cursor + try: + # call Step_5, line 224 + if not self.r_Step_5(): + raise lab14() + except lab14: pass + self.cursor = self.limit - v_12 + except lab7: pass + self.cursor = self.limit_backward + # do, line 227 + v_13 = self.cursor + try: + # call postlude, line 227 + if not self.r_postlude(): + raise lab15() + except lab15: pass + self.cursor = v_13 + except lab0: pass + return True + + def equals(self, o): + return isinstance(o, EnglishStemmer) + + def hashCode(self): + return hash("EnglishStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass diff --git a/contrib/snowballstemmer/finnish_stemmer.py b/contrib/snowballstemmer/finnish_stemmer.py new file mode 100644 index 0000000..8ec540c --- /dev/null +++ b/contrib/snowballstemmer/finnish_stemmer.py @@ -0,0 +1,853 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class FinnishStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"pa", -1, 1), + Among(u"sti", -1, 2), + Among(u"kaan", -1, 1), + Among(u"han", -1, 1), + Among(u"kin", -1, 1), + Among(u"h\u00E4n", -1, 1), + Among(u"k\u00E4\u00E4n", -1, 1), + Among(u"ko", -1, 1), + Among(u"p\u00E4", -1, 1), + Among(u"k\u00F6", -1, 1) + ] + + a_1 = [ + Among(u"lla", -1, -1), + Among(u"na", -1, -1), + Among(u"ssa", -1, -1), + Among(u"ta", -1, -1), + Among(u"lta", 3, -1), + Among(u"sta", 3, -1) + ] + + a_2 = [ + Among(u"ll\u00E4", -1, -1), + Among(u"n\u00E4", -1, -1), + Among(u"ss\u00E4", -1, -1), + Among(u"t\u00E4", -1, -1), + Among(u"lt\u00E4", 3, -1), + Among(u"st\u00E4", 3, -1) + ] + + a_3 = [ + Among(u"lle", -1, -1), + Among(u"ine", -1, -1) + ] + + a_4 = [ + Among(u"nsa", -1, 3), + Among(u"mme", -1, 3), + Among(u"nne", -1, 3), + Among(u"ni", -1, 2), + Among(u"si", -1, 1), + Among(u"an", -1, 4), + Among(u"en", -1, 6), + Among(u"\u00E4n", -1, 5), + Among(u"ns\u00E4", -1, 3) + ] + + a_5 = [ + Among(u"aa", -1, -1), + Among(u"ee", -1, -1), + Among(u"ii", -1, -1), + Among(u"oo", -1, -1), + Among(u"uu", -1, -1), + Among(u"\u00E4\u00E4", -1, -1), + Among(u"\u00F6\u00F6", -1, -1) + ] + + a_6 = [ + Among(u"a", -1, 8), + Among(u"lla", 0, -1), + Among(u"na", 0, -1), + Among(u"ssa", 0, -1), + Among(u"ta", 0, -1), + Among(u"lta", 4, -1), + Among(u"sta", 4, -1), + Among(u"tta", 4, 9), + Among(u"lle", -1, -1), + Among(u"ine", -1, -1), + Among(u"ksi", -1, -1), + Among(u"n", -1, 7), + Among(u"han", 11, 1), + Among(u"den", 11, -1, "r_VI"), + Among(u"seen", 11, -1, "r_LONG"), + Among(u"hen", 11, 2), + Among(u"tten", 11, -1, "r_VI"), + Among(u"hin", 11, 3), + Among(u"siin", 11, -1, "r_VI"), + Among(u"hon", 11, 4), + Among(u"h\u00E4n", 11, 5), + Among(u"h\u00F6n", 11, 6), + Among(u"\u00E4", -1, 8), + Among(u"ll\u00E4", 22, -1), + Among(u"n\u00E4", 22, -1), + Among(u"ss\u00E4", 22, -1), + Among(u"t\u00E4", 22, -1), + Among(u"lt\u00E4", 26, -1), + Among(u"st\u00E4", 26, -1), + Among(u"tt\u00E4", 26, 9) + ] + + a_7 = [ + Among(u"eja", -1, -1), + Among(u"mma", -1, 1), + Among(u"imma", 1, -1), + Among(u"mpa", -1, 1), + Among(u"impa", 3, -1), + Among(u"mmi", -1, 1), + Among(u"immi", 5, -1), + Among(u"mpi", -1, 1), + Among(u"impi", 7, -1), + Among(u"ej\u00E4", -1, -1), + Among(u"mm\u00E4", -1, 1), + Among(u"imm\u00E4", 10, -1), + Among(u"mp\u00E4", -1, 1), + Among(u"imp\u00E4", 12, -1) + ] + + a_8 = [ + Among(u"i", -1, -1), + Among(u"j", -1, -1) + ] + + a_9 = [ + Among(u"mma", -1, 1), + Among(u"imma", 0, -1) + ] + + g_AEI = [17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8] + + g_V1 = [17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32] + + g_V2 = [17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32] + + g_particle_end = [17, 97, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32] + + B_ending_removed = False + S_x = "" + I_p2 = 0 + I_p1 = 0 + + def copy_from(self, other): + self.B_ending_removed = other.B_ending_removed + self.S_x = other.S_x + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + super.copy_from(other) + + + def r_mark_regions(self): + # (, line 41 + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # goto, line 46 + try: + while True: + v_1 = self.cursor + try: + if not self.in_grouping(FinnishStemmer.g_V1, 97, 246): + raise lab1() + self.cursor = v_1 + raise lab0() + except lab1: pass + self.cursor = v_1 + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab0: pass + # gopast, line 46 + try: + while True: + try: + if not self.out_grouping(FinnishStemmer.g_V1, 97, 246): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab2: pass + # setmark p1, line 46 + self.I_p1 = self.cursor + # goto, line 47 + try: + while True: + v_3 = self.cursor + try: + if not self.in_grouping(FinnishStemmer.g_V1, 97, 246): + raise lab5() + self.cursor = v_3 + raise lab4() + except lab5: pass + self.cursor = v_3 + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab4: pass + # gopast, line 47 + try: + while True: + try: + if not self.out_grouping(FinnishStemmer.g_V1, 97, 246): + raise lab7() + raise lab6() + except lab7: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab6: pass + # setmark p2, line 47 + self.I_p2 = self.cursor + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_particle_etc(self): + # (, line 54 + # setlimit, line 55 + v_1 = self.limit - self.cursor + # tomark, line 55 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 55 + # [, line 55 + self.ket = self.cursor + # substring, line 55 + among_var = self.find_among_b(FinnishStemmer.a_0, 10) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 55 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 62 + if not self.in_grouping_b(FinnishStemmer.g_particle_end, 97, 246): + return False + elif among_var == 2: + # (, line 64 + # call R2, line 64 + if not self.r_R2(): + return False + # delete, line 66 + if not self.slice_del(): + return False + + return True + + def r_possessive(self): + # (, line 68 + # setlimit, line 69 + v_1 = self.limit - self.cursor + # tomark, line 69 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 69 + # [, line 69 + self.ket = self.cursor + # substring, line 69 + among_var = self.find_among_b(FinnishStemmer.a_4, 9) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 69 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 72 + # not, line 72 + v_3 = self.limit - self.cursor + try: + # literal, line 72 + if not self.eq_s_b(1, u"k"): + raise lab0() + return False + except lab0: pass + self.cursor = self.limit - v_3 + # delete, line 72 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 74 + # delete, line 74 + if not self.slice_del(): + return False + + # [, line 74 + self.ket = self.cursor + # literal, line 74 + if not self.eq_s_b(3, u"kse"): + return False + # ], line 74 + self.bra = self.cursor + # <-, line 74 + if not self.slice_from(u"ksi"): + return False + elif among_var == 3: + # (, line 78 + # delete, line 78 + if not self.slice_del(): + return False + + elif among_var == 4: + # (, line 81 + # among, line 81 + if self.find_among_b(FinnishStemmer.a_1, 6) == 0: + return False + # delete, line 81 + if not self.slice_del(): + return False + + elif among_var == 5: + # (, line 83 + # among, line 83 + if self.find_among_b(FinnishStemmer.a_2, 6) == 0: + return False + # delete, line 84 + if not self.slice_del(): + return False + + elif among_var == 6: + # (, line 86 + # among, line 86 + if self.find_among_b(FinnishStemmer.a_3, 2) == 0: + return False + # delete, line 86 + if not self.slice_del(): + return False + + return True + + def r_LONG(self): + # among, line 91 + if self.find_among_b(FinnishStemmer.a_5, 7) == 0: + return False + return True + + def r_VI(self): + # (, line 93 + # literal, line 93 + if not self.eq_s_b(1, u"i"): + return False + if not self.in_grouping_b(FinnishStemmer.g_V2, 97, 246): + return False + return True + + def r_case_ending(self): + # (, line 95 + # setlimit, line 96 + v_1 = self.limit - self.cursor + # tomark, line 96 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 96 + # [, line 96 + self.ket = self.cursor + # substring, line 96 + among_var = self.find_among_b(FinnishStemmer.a_6, 30) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 96 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 98 + # literal, line 98 + if not self.eq_s_b(1, u"a"): + return False + elif among_var == 2: + # (, line 99 + # literal, line 99 + if not self.eq_s_b(1, u"e"): + return False + elif among_var == 3: + # (, line 100 + # literal, line 100 + if not self.eq_s_b(1, u"i"): + return False + elif among_var == 4: + # (, line 101 + # literal, line 101 + if not self.eq_s_b(1, u"o"): + return False + elif among_var == 5: + # (, line 102 + # literal, line 102 + if not self.eq_s_b(1, u"\u00E4"): + return False + elif among_var == 6: + # (, line 103 + # literal, line 103 + if not self.eq_s_b(1, u"\u00F6"): + return False + elif among_var == 7: + # (, line 111 + # try, line 111 + v_3 = self.limit - self.cursor + try: + # (, line 111 + # and, line 113 + v_4 = self.limit - self.cursor + # or, line 112 + try: + v_5 = self.limit - self.cursor + try: + # call LONG, line 111 + if not self.r_LONG(): + raise lab2() + raise lab1() + except lab2: pass + self.cursor = self.limit - v_5 + # literal, line 112 + if not self.eq_s_b(2, u"ie"): + self.cursor = self.limit - v_3 + raise lab0() + except lab1: pass + self.cursor = self.limit - v_4 + # next, line 113 + if self.cursor <= self.limit_backward: + self.cursor = self.limit - v_3 + raise lab0() + self.cursor -= 1 + # ], line 113 + self.bra = self.cursor + except lab0: pass + elif among_var == 8: + # (, line 119 + if not self.in_grouping_b(FinnishStemmer.g_V1, 97, 246): + return False + if not self.out_grouping_b(FinnishStemmer.g_V1, 97, 246): + return False + elif among_var == 9: + # (, line 121 + # literal, line 121 + if not self.eq_s_b(1, u"e"): + return False + # delete, line 138 + if not self.slice_del(): + return False + + # set ending_removed, line 139 + self.B_ending_removed = True + return True + + def r_other_endings(self): + # (, line 141 + # setlimit, line 142 + v_1 = self.limit - self.cursor + # tomark, line 142 + if self.cursor < self.I_p2: + return False + self.cursor = self.I_p2 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 142 + # [, line 142 + self.ket = self.cursor + # substring, line 142 + among_var = self.find_among_b(FinnishStemmer.a_7, 14) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 142 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 146 + # not, line 146 + v_3 = self.limit - self.cursor + try: + # literal, line 146 + if not self.eq_s_b(2, u"po"): + raise lab0() + return False + except lab0: pass + self.cursor = self.limit - v_3 + # delete, line 151 + if not self.slice_del(): + return False + + return True + + def r_i_plural(self): + # (, line 153 + # setlimit, line 154 + v_1 = self.limit - self.cursor + # tomark, line 154 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 154 + # [, line 154 + self.ket = self.cursor + # substring, line 154 + if self.find_among_b(FinnishStemmer.a_8, 2) == 0: + self.limit_backward = v_2 + return False + # ], line 154 + self.bra = self.cursor + self.limit_backward = v_2 + # delete, line 158 + if not self.slice_del(): + return False + + return True + + def r_t_plural(self): + # (, line 160 + # setlimit, line 161 + v_1 = self.limit - self.cursor + # tomark, line 161 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 161 + # [, line 162 + self.ket = self.cursor + # literal, line 162 + if not self.eq_s_b(1, u"t"): + self.limit_backward = v_2 + return False + # ], line 162 + self.bra = self.cursor + # test, line 162 + v_3 = self.limit - self.cursor + if not self.in_grouping_b(FinnishStemmer.g_V1, 97, 246): + self.limit_backward = v_2 + return False + self.cursor = self.limit - v_3 + # delete, line 163 + if not self.slice_del(): + return False + + self.limit_backward = v_2 + # setlimit, line 165 + v_4 = self.limit - self.cursor + # tomark, line 165 + if self.cursor < self.I_p2: + return False + self.cursor = self.I_p2 + v_5 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_4 + # (, line 165 + # [, line 165 + self.ket = self.cursor + # substring, line 165 + among_var = self.find_among_b(FinnishStemmer.a_9, 2) + if among_var == 0: + self.limit_backward = v_5 + return False + # ], line 165 + self.bra = self.cursor + self.limit_backward = v_5 + if among_var == 0: + return False + elif among_var == 1: + # (, line 167 + # not, line 167 + v_6 = self.limit - self.cursor + try: + # literal, line 167 + if not self.eq_s_b(2, u"po"): + raise lab0() + return False + except lab0: pass + self.cursor = self.limit - v_6 + # delete, line 170 + if not self.slice_del(): + return False + + return True + + def r_tidy(self): + # (, line 172 + # setlimit, line 173 + v_1 = self.limit - self.cursor + # tomark, line 173 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 173 + # do, line 174 + v_3 = self.limit - self.cursor + try: + # (, line 174 + # and, line 174 + v_4 = self.limit - self.cursor + # call LONG, line 174 + if not self.r_LONG(): + raise lab0() + self.cursor = self.limit - v_4 + # (, line 174 + # [, line 174 + self.ket = self.cursor + # next, line 174 + if self.cursor <= self.limit_backward: + raise lab0() + self.cursor -= 1 + # ], line 174 + self.bra = self.cursor + # delete, line 174 + if not self.slice_del(): + return False + + except lab0: pass + self.cursor = self.limit - v_3 + # do, line 175 + v_5 = self.limit - self.cursor + try: + # (, line 175 + # [, line 175 + self.ket = self.cursor + if not self.in_grouping_b(FinnishStemmer.g_AEI, 97, 228): + raise lab1() + # ], line 175 + self.bra = self.cursor + if not self.out_grouping_b(FinnishStemmer.g_V1, 97, 246): + raise lab1() + # delete, line 175 + if not self.slice_del(): + return False + + except lab1: pass + self.cursor = self.limit - v_5 + # do, line 176 + v_6 = self.limit - self.cursor + try: + # (, line 176 + # [, line 176 + self.ket = self.cursor + # literal, line 176 + if not self.eq_s_b(1, u"j"): + raise lab2() + # ], line 176 + self.bra = self.cursor + # or, line 176 + try: + v_7 = self.limit - self.cursor + try: + # literal, line 176 + if not self.eq_s_b(1, u"o"): + raise lab4() + raise lab3() + except lab4: pass + self.cursor = self.limit - v_7 + # literal, line 176 + if not self.eq_s_b(1, u"u"): + raise lab2() + except lab3: pass + # delete, line 176 + if not self.slice_del(): + return False + + except lab2: pass + self.cursor = self.limit - v_6 + # do, line 177 + v_8 = self.limit - self.cursor + try: + # (, line 177 + # [, line 177 + self.ket = self.cursor + # literal, line 177 + if not self.eq_s_b(1, u"o"): + raise lab5() + # ], line 177 + self.bra = self.cursor + # literal, line 177 + if not self.eq_s_b(1, u"j"): + raise lab5() + # delete, line 177 + if not self.slice_del(): + return False + + except lab5: pass + self.cursor = self.limit - v_8 + self.limit_backward = v_2 + # goto, line 179 + try: + while True: + v_9 = self.limit - self.cursor + try: + if not self.out_grouping_b(FinnishStemmer.g_V1, 97, 246): + raise lab7() + self.cursor = self.limit - v_9 + raise lab6() + except lab7: pass + self.cursor = self.limit - v_9 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab6: pass + # [, line 179 + self.ket = self.cursor + # next, line 179 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # ], line 179 + self.bra = self.cursor + # -> x, line 179 + self.S_x = self.slice_to(self.S_x) + if self.S_x == '': + return False + # name x, line 179 + if not self.eq_v_b(self.S_x): + return False + # delete, line 179 + if not self.slice_del(): + return False + + return True + + def _stem(self): + # (, line 183 + # do, line 185 + v_1 = self.cursor + try: + # call mark_regions, line 185 + if not self.r_mark_regions(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # unset ending_removed, line 186 + self.B_ending_removed = False + # backwards, line 187 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 187 + # do, line 188 + v_2 = self.limit - self.cursor + try: + # call particle_etc, line 188 + if not self.r_particle_etc(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 189 + v_3 = self.limit - self.cursor + try: + # call possessive, line 189 + if not self.r_possessive(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 190 + v_4 = self.limit - self.cursor + try: + # call case_ending, line 190 + if not self.r_case_ending(): + raise lab3() + except lab3: pass + self.cursor = self.limit - v_4 + # do, line 191 + v_5 = self.limit - self.cursor + try: + # call other_endings, line 191 + if not self.r_other_endings(): + raise lab4() + except lab4: pass + self.cursor = self.limit - v_5 + # or, line 192 + try: + v_6 = self.limit - self.cursor + try: + # (, line 192 + # Boolean test ending_removed, line 192 + if not self.B_ending_removed: + raise lab6() + # do, line 192 + v_7 = self.limit - self.cursor + try: + # call i_plural, line 192 + if not self.r_i_plural(): + raise lab7() + except lab7: pass + self.cursor = self.limit - v_7 + raise lab5() + except lab6: pass + self.cursor = self.limit - v_6 + # do, line 192 + v_8 = self.limit - self.cursor + try: + # call t_plural, line 192 + if not self.r_t_plural(): + raise lab8() + except lab8: pass + self.cursor = self.limit - v_8 + except lab5: pass + # do, line 193 + v_9 = self.limit - self.cursor + try: + # call tidy, line 193 + if not self.r_tidy(): + raise lab9() + except lab9: pass + self.cursor = self.limit - v_9 + self.cursor = self.limit_backward + return True + + def equals(self, o): + return isinstance(o, FinnishStemmer) + + def hashCode(self): + return hash("FinnishStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass diff --git a/contrib/snowballstemmer/french_stemmer.py b/contrib/snowballstemmer/french_stemmer.py new file mode 100644 index 0000000..4c3772a --- /dev/null +++ b/contrib/snowballstemmer/french_stemmer.py @@ -0,0 +1,1307 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class FrenchStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"col", -1, -1), + Among(u"par", -1, -1), + Among(u"tap", -1, -1) + ] + + a_1 = [ + Among(u"", -1, 4), + Among(u"I", 0, 1), + Among(u"U", 0, 2), + Among(u"Y", 0, 3) + ] + + a_2 = [ + Among(u"iqU", -1, 3), + Among(u"abl", -1, 3), + Among(u"I\u00E8r", -1, 4), + Among(u"i\u00E8r", -1, 4), + Among(u"eus", -1, 2), + Among(u"iv", -1, 1) + ] + + a_3 = [ + Among(u"ic", -1, 2), + Among(u"abil", -1, 1), + Among(u"iv", -1, 3) + ] + + a_4 = [ + Among(u"iqUe", -1, 1), + Among(u"atrice", -1, 2), + Among(u"ance", -1, 1), + Among(u"ence", -1, 5), + Among(u"logie", -1, 3), + Among(u"able", -1, 1), + Among(u"isme", -1, 1), + Among(u"euse", -1, 11), + Among(u"iste", -1, 1), + Among(u"ive", -1, 8), + Among(u"if", -1, 8), + Among(u"usion", -1, 4), + Among(u"ation", -1, 2), + Among(u"ution", -1, 4), + Among(u"ateur", -1, 2), + Among(u"iqUes", -1, 1), + Among(u"atrices", -1, 2), + Among(u"ances", -1, 1), + Among(u"ences", -1, 5), + Among(u"logies", -1, 3), + Among(u"ables", -1, 1), + Among(u"ismes", -1, 1), + Among(u"euses", -1, 11), + Among(u"istes", -1, 1), + Among(u"ives", -1, 8), + Among(u"ifs", -1, 8), + Among(u"usions", -1, 4), + Among(u"ations", -1, 2), + Among(u"utions", -1, 4), + Among(u"ateurs", -1, 2), + Among(u"ments", -1, 15), + Among(u"ements", 30, 6), + Among(u"issements", 31, 12), + Among(u"it\u00E9s", -1, 7), + Among(u"ment", -1, 15), + Among(u"ement", 34, 6), + Among(u"issement", 35, 12), + Among(u"amment", 34, 13), + Among(u"emment", 34, 14), + Among(u"aux", -1, 10), + Among(u"eaux", 39, 9), + Among(u"eux", -1, 1), + Among(u"it\u00E9", -1, 7) + ] + + a_5 = [ + Among(u"ira", -1, 1), + Among(u"ie", -1, 1), + Among(u"isse", -1, 1), + Among(u"issante", -1, 1), + Among(u"i", -1, 1), + Among(u"irai", 4, 1), + Among(u"ir", -1, 1), + Among(u"iras", -1, 1), + Among(u"ies", -1, 1), + Among(u"\u00EEmes", -1, 1), + Among(u"isses", -1, 1), + Among(u"issantes", -1, 1), + Among(u"\u00EEtes", -1, 1), + Among(u"is", -1, 1), + Among(u"irais", 13, 1), + Among(u"issais", 13, 1), + Among(u"irions", -1, 1), + Among(u"issions", -1, 1), + Among(u"irons", -1, 1), + Among(u"issons", -1, 1), + Among(u"issants", -1, 1), + Among(u"it", -1, 1), + Among(u"irait", 21, 1), + Among(u"issait", 21, 1), + Among(u"issant", -1, 1), + Among(u"iraIent", -1, 1), + Among(u"issaIent", -1, 1), + Among(u"irent", -1, 1), + Among(u"issent", -1, 1), + Among(u"iront", -1, 1), + Among(u"\u00EEt", -1, 1), + Among(u"iriez", -1, 1), + Among(u"issiez", -1, 1), + Among(u"irez", -1, 1), + Among(u"issez", -1, 1) + ] + + a_6 = [ + Among(u"a", -1, 3), + Among(u"era", 0, 2), + Among(u"asse", -1, 3), + Among(u"ante", -1, 3), + Among(u"\u00E9e", -1, 2), + Among(u"ai", -1, 3), + Among(u"erai", 5, 2), + Among(u"er", -1, 2), + Among(u"as", -1, 3), + Among(u"eras", 8, 2), + Among(u"\u00E2mes", -1, 3), + Among(u"asses", -1, 3), + Among(u"antes", -1, 3), + Among(u"\u00E2tes", -1, 3), + Among(u"\u00E9es", -1, 2), + Among(u"ais", -1, 3), + Among(u"erais", 15, 2), + Among(u"ions", -1, 1), + Among(u"erions", 17, 2), + Among(u"assions", 17, 3), + Among(u"erons", -1, 2), + Among(u"ants", -1, 3), + Among(u"\u00E9s", -1, 2), + Among(u"ait", -1, 3), + Among(u"erait", 23, 2), + Among(u"ant", -1, 3), + Among(u"aIent", -1, 3), + Among(u"eraIent", 26, 2), + Among(u"\u00E8rent", -1, 2), + Among(u"assent", -1, 3), + Among(u"eront", -1, 2), + Among(u"\u00E2t", -1, 3), + Among(u"ez", -1, 2), + Among(u"iez", 32, 2), + Among(u"eriez", 33, 2), + Among(u"assiez", 33, 3), + Among(u"erez", 32, 2), + Among(u"\u00E9", -1, 2) + ] + + a_7 = [ + Among(u"e", -1, 3), + Among(u"I\u00E8re", 0, 2), + Among(u"i\u00E8re", 0, 2), + Among(u"ion", -1, 1), + Among(u"Ier", -1, 2), + Among(u"ier", -1, 2), + Among(u"\u00EB", -1, 4) + ] + + a_8 = [ + Among(u"ell", -1, -1), + Among(u"eill", -1, -1), + Among(u"enn", -1, -1), + Among(u"onn", -1, -1), + Among(u"ett", -1, -1) + ] + + g_v = [17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 130, 103, 8, 5] + + g_keep_with_s = [1, 65, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128] + + I_p2 = 0 + I_p1 = 0 + I_pV = 0 + + def copy_from(self, other): + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + self.I_pV = other.I_pV + super.copy_from(other) + + + def r_prelude(self): + # repeat, line 38 + try: + while True: + try: + v_1 = self.cursor + try: + # goto, line 38 + try: + while True: + v_2 = self.cursor + try: + # (, line 38 + # or, line 44 + try: + v_3 = self.cursor + try: + # (, line 40 + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab6() + # [, line 40 + self.bra = self.cursor + # or, line 40 + try: + v_4 = self.cursor + try: + # (, line 40 + # literal, line 40 + if not self.eq_s(1, u"u"): + raise lab8() + # ], line 40 + self.ket = self.cursor + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab8() + # <-, line 40 + if not self.slice_from(u"U"): + return False + raise lab7() + except lab8: pass + self.cursor = v_4 + try: + # (, line 41 + # literal, line 41 + if not self.eq_s(1, u"i"): + raise lab9() + # ], line 41 + self.ket = self.cursor + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab9() + # <-, line 41 + if not self.slice_from(u"I"): + return False + raise lab7() + except lab9: pass + self.cursor = v_4 + # (, line 42 + # literal, line 42 + if not self.eq_s(1, u"y"): + raise lab6() + # ], line 42 + self.ket = self.cursor + # <-, line 42 + if not self.slice_from(u"Y"): + return False + except lab7: pass + raise lab5() + except lab6: pass + self.cursor = v_3 + try: + # (, line 45 + # [, line 45 + self.bra = self.cursor + # literal, line 45 + if not self.eq_s(1, u"y"): + raise lab10() + # ], line 45 + self.ket = self.cursor + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab10() + # <-, line 45 + if not self.slice_from(u"Y"): + return False + raise lab5() + except lab10: pass + self.cursor = v_3 + # (, line 47 + # literal, line 47 + if not self.eq_s(1, u"q"): + raise lab4() + # [, line 47 + self.bra = self.cursor + # literal, line 47 + if not self.eq_s(1, u"u"): + raise lab4() + # ], line 47 + self.ket = self.cursor + # <-, line 47 + if not self.slice_from(u"U"): + return False + except lab5: pass + self.cursor = v_2 + raise lab3() + except lab4: pass + self.cursor = v_2 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab3: pass + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_mark_regions(self): + # (, line 50 + self.I_pV = self.limit; + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # do, line 56 + v_1 = self.cursor + try: + # (, line 56 + # or, line 58 + try: + v_2 = self.cursor + try: + # (, line 57 + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab2() + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab2() + # next, line 57 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_2 + try: + # among, line 59 + if self.find_among(FrenchStemmer.a_0, 3) == 0: + raise lab3() + raise lab1() + except lab3: pass + self.cursor = v_2 + # (, line 66 + # next, line 66 + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + # gopast, line 66 + try: + while True: + try: + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab5() + raise lab4() + except lab5: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab4: pass + except lab1: pass + # setmark pV, line 67 + self.I_pV = self.cursor + except lab0: pass + self.cursor = v_1 + # do, line 69 + v_4 = self.cursor + try: + # (, line 69 + # gopast, line 70 + try: + while True: + try: + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + raise lab6() + self.cursor += 1 + except lab7: pass + # gopast, line 70 + try: + while True: + try: + if not self.out_grouping(FrenchStemmer.g_v, 97, 251): + raise lab10() + raise lab9() + except lab10: pass + if self.cursor >= self.limit: + raise lab6() + self.cursor += 1 + except lab9: pass + # setmark p1, line 70 + self.I_p1 = self.cursor + # gopast, line 71 + try: + while True: + try: + if not self.in_grouping(FrenchStemmer.g_v, 97, 251): + raise lab12() + raise lab11() + except lab12: pass + if self.cursor >= self.limit: + raise lab6() + self.cursor += 1 + except lab11: pass + # gopast, line 71 + try: + while True: + try: + if not self.out_grouping(FrenchStemmer.g_v, 97, 251): + raise lab14() + raise lab13() + except lab14: pass + if self.cursor >= self.limit: + raise lab6() + self.cursor += 1 + except lab13: pass + # setmark p2, line 71 + self.I_p2 = self.cursor + except lab6: pass + self.cursor = v_4 + return True + + def r_postlude(self): + # repeat, line 75 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 75 + # [, line 77 + self.bra = self.cursor + # substring, line 77 + among_var = self.find_among(FrenchStemmer.a_1, 4) + if among_var == 0: + raise lab2() + # ], line 77 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 78 + # <-, line 78 + if not self.slice_from(u"i"): + return False + elif among_var == 2: + # (, line 79 + # <-, line 79 + if not self.slice_from(u"u"): + return False + elif among_var == 3: + # (, line 80 + # <-, line 80 + if not self.slice_from(u"y"): + return False + elif among_var == 4: + # (, line 81 + # next, line 81 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_RV(self): + if not self.I_pV <= self.cursor: + return False + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_standard_suffix(self): + # (, line 91 + # [, line 92 + self.ket = self.cursor + # substring, line 92 + among_var = self.find_among_b(FrenchStemmer.a_4, 43) + if among_var == 0: + return False + # ], line 92 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 96 + # call R2, line 96 + if not self.r_R2(): + return False + # delete, line 96 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 99 + # call R2, line 99 + if not self.r_R2(): + return False + # delete, line 99 + if not self.slice_del(): + return False + + # try, line 100 + v_1 = self.limit - self.cursor + try: + # (, line 100 + # [, line 100 + self.ket = self.cursor + # literal, line 100 + if not self.eq_s_b(2, u"ic"): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 100 + self.bra = self.cursor + # or, line 100 + try: + v_2 = self.limit - self.cursor + try: + # (, line 100 + # call R2, line 100 + if not self.r_R2(): + raise lab2() + # delete, line 100 + if not self.slice_del(): + return False + + raise lab1() + except lab2: pass + self.cursor = self.limit - v_2 + # <-, line 100 + if not self.slice_from(u"iqU"): + return False + except lab1: pass + except lab0: pass + elif among_var == 3: + # (, line 104 + # call R2, line 104 + if not self.r_R2(): + return False + # <-, line 104 + if not self.slice_from(u"log"): + return False + elif among_var == 4: + # (, line 107 + # call R2, line 107 + if not self.r_R2(): + return False + # <-, line 107 + if not self.slice_from(u"u"): + return False + elif among_var == 5: + # (, line 110 + # call R2, line 110 + if not self.r_R2(): + return False + # <-, line 110 + if not self.slice_from(u"ent"): + return False + elif among_var == 6: + # (, line 113 + # call RV, line 114 + if not self.r_RV(): + return False + # delete, line 114 + if not self.slice_del(): + return False + + # try, line 115 + v_3 = self.limit - self.cursor + try: + # (, line 115 + # [, line 116 + self.ket = self.cursor + # substring, line 116 + among_var = self.find_among_b(FrenchStemmer.a_2, 6) + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab3() + # ], line 116 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab3() + elif among_var == 1: + # (, line 117 + # call R2, line 117 + if not self.r_R2(): + self.cursor = self.limit - v_3 + raise lab3() + # delete, line 117 + if not self.slice_del(): + return False + + # [, line 117 + self.ket = self.cursor + # literal, line 117 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_3 + raise lab3() + # ], line 117 + self.bra = self.cursor + # call R2, line 117 + if not self.r_R2(): + self.cursor = self.limit - v_3 + raise lab3() + # delete, line 117 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 118 + # or, line 118 + try: + v_4 = self.limit - self.cursor + try: + # (, line 118 + # call R2, line 118 + if not self.r_R2(): + raise lab5() + # delete, line 118 + if not self.slice_del(): + return False + + raise lab4() + except lab5: pass + self.cursor = self.limit - v_4 + # (, line 118 + # call R1, line 118 + if not self.r_R1(): + self.cursor = self.limit - v_3 + raise lab3() + # <-, line 118 + if not self.slice_from(u"eux"): + return False + except lab4: pass + elif among_var == 3: + # (, line 120 + # call R2, line 120 + if not self.r_R2(): + self.cursor = self.limit - v_3 + raise lab3() + # delete, line 120 + if not self.slice_del(): + return False + + elif among_var == 4: + # (, line 122 + # call RV, line 122 + if not self.r_RV(): + self.cursor = self.limit - v_3 + raise lab3() + # <-, line 122 + if not self.slice_from(u"i"): + return False + except lab3: pass + elif among_var == 7: + # (, line 128 + # call R2, line 129 + if not self.r_R2(): + return False + # delete, line 129 + if not self.slice_del(): + return False + + # try, line 130 + v_5 = self.limit - self.cursor + try: + # (, line 130 + # [, line 131 + self.ket = self.cursor + # substring, line 131 + among_var = self.find_among_b(FrenchStemmer.a_3, 3) + if among_var == 0: + self.cursor = self.limit - v_5 + raise lab6() + # ], line 131 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_5 + raise lab6() + elif among_var == 1: + # (, line 132 + # or, line 132 + try: + v_6 = self.limit - self.cursor + try: + # (, line 132 + # call R2, line 132 + if not self.r_R2(): + raise lab8() + # delete, line 132 + if not self.slice_del(): + return False + + raise lab7() + except lab8: pass + self.cursor = self.limit - v_6 + # <-, line 132 + if not self.slice_from(u"abl"): + return False + except lab7: pass + elif among_var == 2: + # (, line 133 + # or, line 133 + try: + v_7 = self.limit - self.cursor + try: + # (, line 133 + # call R2, line 133 + if not self.r_R2(): + raise lab10() + # delete, line 133 + if not self.slice_del(): + return False + + raise lab9() + except lab10: pass + self.cursor = self.limit - v_7 + # <-, line 133 + if not self.slice_from(u"iqU"): + return False + except lab9: pass + elif among_var == 3: + # (, line 134 + # call R2, line 134 + if not self.r_R2(): + self.cursor = self.limit - v_5 + raise lab6() + # delete, line 134 + if not self.slice_del(): + return False + + except lab6: pass + elif among_var == 8: + # (, line 140 + # call R2, line 141 + if not self.r_R2(): + return False + # delete, line 141 + if not self.slice_del(): + return False + + # try, line 142 + v_8 = self.limit - self.cursor + try: + # (, line 142 + # [, line 142 + self.ket = self.cursor + # literal, line 142 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_8 + raise lab11() + # ], line 142 + self.bra = self.cursor + # call R2, line 142 + if not self.r_R2(): + self.cursor = self.limit - v_8 + raise lab11() + # delete, line 142 + if not self.slice_del(): + return False + + # [, line 142 + self.ket = self.cursor + # literal, line 142 + if not self.eq_s_b(2, u"ic"): + self.cursor = self.limit - v_8 + raise lab11() + # ], line 142 + self.bra = self.cursor + # or, line 142 + try: + v_9 = self.limit - self.cursor + try: + # (, line 142 + # call R2, line 142 + if not self.r_R2(): + raise lab13() + # delete, line 142 + if not self.slice_del(): + return False + + raise lab12() + except lab13: pass + self.cursor = self.limit - v_9 + # <-, line 142 + if not self.slice_from(u"iqU"): + return False + except lab12: pass + except lab11: pass + elif among_var == 9: + # (, line 144 + # <-, line 144 + if not self.slice_from(u"eau"): + return False + elif among_var == 10: + # (, line 145 + # call R1, line 145 + if not self.r_R1(): + return False + # <-, line 145 + if not self.slice_from(u"al"): + return False + elif among_var == 11: + # (, line 147 + # or, line 147 + try: + v_10 = self.limit - self.cursor + try: + # (, line 147 + # call R2, line 147 + if not self.r_R2(): + raise lab15() + # delete, line 147 + if not self.slice_del(): + return False + + raise lab14() + except lab15: pass + self.cursor = self.limit - v_10 + # (, line 147 + # call R1, line 147 + if not self.r_R1(): + return False + # <-, line 147 + if not self.slice_from(u"eux"): + return False + except lab14: pass + elif among_var == 12: + # (, line 150 + # call R1, line 150 + if not self.r_R1(): + return False + if not self.out_grouping_b(FrenchStemmer.g_v, 97, 251): + return False + # delete, line 150 + if not self.slice_del(): + return False + + elif among_var == 13: + # (, line 155 + # call RV, line 155 + if not self.r_RV(): + return False + # fail, line 155 + # (, line 155 + # <-, line 155 + if not self.slice_from(u"ant"): + return False + return False + elif among_var == 14: + # (, line 156 + # call RV, line 156 + if not self.r_RV(): + return False + # fail, line 156 + # (, line 156 + # <-, line 156 + if not self.slice_from(u"ent"): + return False + return False + elif among_var == 15: + # (, line 158 + # test, line 158 + v_11 = self.limit - self.cursor + # (, line 158 + if not self.in_grouping_b(FrenchStemmer.g_v, 97, 251): + return False + # call RV, line 158 + if not self.r_RV(): + return False + self.cursor = self.limit - v_11 + # fail, line 158 + # (, line 158 + # delete, line 158 + if not self.slice_del(): + return False + + return False + return True + + def r_i_verb_suffix(self): + # setlimit, line 163 + v_1 = self.limit - self.cursor + # tomark, line 163 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 163 + # [, line 164 + self.ket = self.cursor + # substring, line 164 + among_var = self.find_among_b(FrenchStemmer.a_5, 35) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 164 + self.bra = self.cursor + if among_var == 0: + self.limit_backward = v_2 + return False + elif among_var == 1: + # (, line 170 + if not self.out_grouping_b(FrenchStemmer.g_v, 97, 251): + self.limit_backward = v_2 + return False + # delete, line 170 + if not self.slice_del(): + return False + + self.limit_backward = v_2 + return True + + def r_verb_suffix(self): + # setlimit, line 174 + v_1 = self.limit - self.cursor + # tomark, line 174 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 174 + # [, line 175 + self.ket = self.cursor + # substring, line 175 + among_var = self.find_among_b(FrenchStemmer.a_6, 38) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 175 + self.bra = self.cursor + if among_var == 0: + self.limit_backward = v_2 + return False + elif among_var == 1: + # (, line 177 + # call R2, line 177 + if not self.r_R2(): + self.limit_backward = v_2 + return False + # delete, line 177 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 185 + # delete, line 185 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 190 + # delete, line 190 + if not self.slice_del(): + return False + + # try, line 191 + v_3 = self.limit - self.cursor + try: + # (, line 191 + # [, line 191 + self.ket = self.cursor + # literal, line 191 + if not self.eq_s_b(1, u"e"): + self.cursor = self.limit - v_3 + raise lab0() + # ], line 191 + self.bra = self.cursor + # delete, line 191 + if not self.slice_del(): + return False + + except lab0: pass + self.limit_backward = v_2 + return True + + def r_residual_suffix(self): + # (, line 198 + # try, line 199 + v_1 = self.limit - self.cursor + try: + # (, line 199 + # [, line 199 + self.ket = self.cursor + # literal, line 199 + if not self.eq_s_b(1, u"s"): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 199 + self.bra = self.cursor + # test, line 199 + v_2 = self.limit - self.cursor + if not self.out_grouping_b(FrenchStemmer.g_keep_with_s, 97, 232): + self.cursor = self.limit - v_1 + raise lab0() + self.cursor = self.limit - v_2 + # delete, line 199 + if not self.slice_del(): + return False + + except lab0: pass + # setlimit, line 200 + v_3 = self.limit - self.cursor + # tomark, line 200 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_4 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_3 + # (, line 200 + # [, line 201 + self.ket = self.cursor + # substring, line 201 + among_var = self.find_among_b(FrenchStemmer.a_7, 7) + if among_var == 0: + self.limit_backward = v_4 + return False + # ], line 201 + self.bra = self.cursor + if among_var == 0: + self.limit_backward = v_4 + return False + elif among_var == 1: + # (, line 202 + # call R2, line 202 + if not self.r_R2(): + self.limit_backward = v_4 + return False + # or, line 202 + try: + v_5 = self.limit - self.cursor + try: + # literal, line 202 + if not self.eq_s_b(1, u"s"): + raise lab2() + raise lab1() + except lab2: pass + self.cursor = self.limit - v_5 + # literal, line 202 + if not self.eq_s_b(1, u"t"): + self.limit_backward = v_4 + return False + except lab1: pass + # delete, line 202 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 204 + # <-, line 204 + if not self.slice_from(u"i"): + return False + elif among_var == 3: + # (, line 205 + # delete, line 205 + if not self.slice_del(): + return False + + elif among_var == 4: + # (, line 206 + # literal, line 206 + if not self.eq_s_b(2, u"gu"): + self.limit_backward = v_4 + return False + # delete, line 206 + if not self.slice_del(): + return False + + self.limit_backward = v_4 + return True + + def r_un_double(self): + # (, line 211 + # test, line 212 + v_1 = self.limit - self.cursor + # among, line 212 + if self.find_among_b(FrenchStemmer.a_8, 5) == 0: + return False + self.cursor = self.limit - v_1 + # [, line 212 + self.ket = self.cursor + # next, line 212 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # ], line 212 + self.bra = self.cursor + # delete, line 212 + if not self.slice_del(): + return False + + return True + + def r_un_accent(self): + # (, line 215 + # atleast, line 216 + v_1 = 1 + # atleast, line 216 + try: + while True: + try: + try: + if not self.out_grouping_b(FrenchStemmer.g_v, 97, 251): + raise lab2() + v_1 -= 1 + raise lab1() + except lab2: pass + raise lab0() + except lab1: pass + except lab0: pass + if v_1 > 0: + return False + # [, line 217 + self.ket = self.cursor + # or, line 217 + try: + v_3 = self.limit - self.cursor + try: + # literal, line 217 + if not self.eq_s_b(1, u"\u00E9"): + raise lab4() + raise lab3() + except lab4: pass + self.cursor = self.limit - v_3 + # literal, line 217 + if not self.eq_s_b(1, u"\u00E8"): + return False + except lab3: pass + # ], line 217 + self.bra = self.cursor + # <-, line 217 + if not self.slice_from(u"e"): + return False + return True + + def _stem(self): + # (, line 221 + # do, line 223 + v_1 = self.cursor + try: + # call prelude, line 223 + if not self.r_prelude(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # do, line 224 + v_2 = self.cursor + try: + # call mark_regions, line 224 + if not self.r_mark_regions(): + raise lab1() + except lab1: pass + self.cursor = v_2 + # backwards, line 225 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 225 + # do, line 227 + v_3 = self.limit - self.cursor + try: + # (, line 227 + # or, line 237 + try: + v_4 = self.limit - self.cursor + try: + # (, line 228 + # and, line 233 + v_5 = self.limit - self.cursor + # (, line 229 + # or, line 229 + try: + v_6 = self.limit - self.cursor + try: + # call standard_suffix, line 229 + if not self.r_standard_suffix(): + raise lab6() + raise lab5() + except lab6: pass + self.cursor = self.limit - v_6 + try: + # call i_verb_suffix, line 230 + if not self.r_i_verb_suffix(): + raise lab7() + raise lab5() + except lab7: pass + self.cursor = self.limit - v_6 + # call verb_suffix, line 231 + if not self.r_verb_suffix(): + raise lab4() + except lab5: pass + self.cursor = self.limit - v_5 + # try, line 234 + v_7 = self.limit - self.cursor + try: + # (, line 234 + # [, line 234 + self.ket = self.cursor + # or, line 234 + try: + v_8 = self.limit - self.cursor + try: + # (, line 234 + # literal, line 234 + if not self.eq_s_b(1, u"Y"): + raise lab10() + # ], line 234 + self.bra = self.cursor + # <-, line 234 + if not self.slice_from(u"i"): + return False + raise lab9() + except lab10: pass + self.cursor = self.limit - v_8 + # (, line 235 + # literal, line 235 + if not self.eq_s_b(1, u"\u00E7"): + self.cursor = self.limit - v_7 + raise lab8() + # ], line 235 + self.bra = self.cursor + # <-, line 235 + if not self.slice_from(u"c"): + return False + except lab9: pass + except lab8: pass + raise lab3() + except lab4: pass + self.cursor = self.limit - v_4 + # call residual_suffix, line 238 + if not self.r_residual_suffix(): + raise lab2() + except lab3: pass + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 243 + v_9 = self.limit - self.cursor + try: + # call un_double, line 243 + if not self.r_un_double(): + raise lab11() + except lab11: pass + self.cursor = self.limit - v_9 + # do, line 244 + v_10 = self.limit - self.cursor + try: + # call un_accent, line 244 + if not self.r_un_accent(): + raise lab12() + except lab12: pass + self.cursor = self.limit - v_10 + self.cursor = self.limit_backward + # do, line 246 + v_11 = self.cursor + try: + # call postlude, line 246 + if not self.r_postlude(): + raise lab13() + except lab13: pass + self.cursor = v_11 + return True + + def equals(self, o): + return isinstance(o, FrenchStemmer) + + def hashCode(self): + return hash("FrenchStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass diff --git a/contrib/snowballstemmer/german_stemmer.py b/contrib/snowballstemmer/german_stemmer.py new file mode 100644 index 0000000..5b7207d --- /dev/null +++ b/contrib/snowballstemmer/german_stemmer.py @@ -0,0 +1,619 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class GermanStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"", -1, 6), + Among(u"U", 0, 2), + Among(u"Y", 0, 1), + Among(u"\u00E4", 0, 3), + Among(u"\u00F6", 0, 4), + Among(u"\u00FC", 0, 5) + ] + + a_1 = [ + Among(u"e", -1, 2), + Among(u"em", -1, 1), + Among(u"en", -1, 2), + Among(u"ern", -1, 1), + Among(u"er", -1, 1), + Among(u"s", -1, 3), + Among(u"es", 5, 2) + ] + + a_2 = [ + Among(u"en", -1, 1), + Among(u"er", -1, 1), + Among(u"st", -1, 2), + Among(u"est", 2, 1) + ] + + a_3 = [ + Among(u"ig", -1, 1), + Among(u"lich", -1, 1) + ] + + a_4 = [ + Among(u"end", -1, 1), + Among(u"ig", -1, 2), + Among(u"ung", -1, 1), + Among(u"lich", -1, 3), + Among(u"isch", -1, 2), + Among(u"ik", -1, 2), + Among(u"heit", -1, 3), + Among(u"keit", -1, 4) + ] + + g_v = [17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32, 8] + + g_s_ending = [117, 30, 5] + + g_st_ending = [117, 30, 4] + + I_x = 0 + I_p2 = 0 + I_p1 = 0 + + def copy_from(self, other): + self.I_x = other.I_x + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + super.copy_from(other) + + + def r_prelude(self): + # (, line 33 + # test, line 35 + v_1 = self.cursor + # repeat, line 35 + try: + while True: + try: + v_2 = self.cursor + try: + # (, line 35 + # or, line 38 + try: + v_3 = self.cursor + try: + # (, line 36 + # [, line 37 + self.bra = self.cursor + # literal, line 37 + if not self.eq_s(1, u"\u00DF"): + raise lab4() + # ], line 37 + self.ket = self.cursor + # <-, line 37 + if not self.slice_from(u"ss"): + return False + raise lab3() + except lab4: pass + self.cursor = v_3 + # next, line 38 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab3: pass + raise lab1() + except lab2: pass + self.cursor = v_2 + raise lab0() + except lab1: pass + except lab0: pass + self.cursor = v_1 + # repeat, line 41 + try: + while True: + try: + v_4 = self.cursor + try: + # goto, line 41 + try: + while True: + v_5 = self.cursor + try: + # (, line 41 + if not self.in_grouping(GermanStemmer.g_v, 97, 252): + raise lab9() + # [, line 42 + self.bra = self.cursor + # or, line 42 + try: + v_6 = self.cursor + try: + # (, line 42 + # literal, line 42 + if not self.eq_s(1, u"u"): + raise lab11() + # ], line 42 + self.ket = self.cursor + if not self.in_grouping(GermanStemmer.g_v, 97, 252): + raise lab11() + # <-, line 42 + if not self.slice_from(u"U"): + return False + raise lab10() + except lab11: pass + self.cursor = v_6 + # (, line 43 + # literal, line 43 + if not self.eq_s(1, u"y"): + raise lab9() + # ], line 43 + self.ket = self.cursor + if not self.in_grouping(GermanStemmer.g_v, 97, 252): + raise lab9() + # <-, line 43 + if not self.slice_from(u"Y"): + return False + except lab10: pass + self.cursor = v_5 + raise lab8() + except lab9: pass + self.cursor = v_5 + if self.cursor >= self.limit: + raise lab7() + self.cursor += 1 + except lab8: pass + raise lab6() + except lab7: pass + self.cursor = v_4 + raise lab5() + except lab6: pass + except lab5: pass + return True + + def r_mark_regions(self): + # (, line 47 + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # test, line 52 + v_1 = self.cursor + # (, line 52 + # hop, line 52 + c = self.cursor + 3 + if 0 > c or c > self.limit: + return False + self.cursor = c + # setmark x, line 52 + self.I_x = self.cursor + self.cursor = v_1 + # gopast, line 54 + try: + while True: + try: + if not self.in_grouping(GermanStemmer.g_v, 97, 252): + raise lab1() + raise lab0() + except lab1: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab0: pass + # gopast, line 54 + try: + while True: + try: + if not self.out_grouping(GermanStemmer.g_v, 97, 252): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab2: pass + # setmark p1, line 54 + self.I_p1 = self.cursor + # try, line 55 + try: + # (, line 55 + if not (self.I_p1 < self.I_x): + raise lab4() + self.I_p1 = self.I_x; + except lab4: pass + # gopast, line 56 + try: + while True: + try: + if not self.in_grouping(GermanStemmer.g_v, 97, 252): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab5: pass + # gopast, line 56 + try: + while True: + try: + if not self.out_grouping(GermanStemmer.g_v, 97, 252): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab7: pass + # setmark p2, line 56 + self.I_p2 = self.cursor + return True + + def r_postlude(self): + # repeat, line 60 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 60 + # [, line 62 + self.bra = self.cursor + # substring, line 62 + among_var = self.find_among(GermanStemmer.a_0, 6) + if among_var == 0: + raise lab2() + # ], line 62 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 63 + # <-, line 63 + if not self.slice_from(u"y"): + return False + elif among_var == 2: + # (, line 64 + # <-, line 64 + if not self.slice_from(u"u"): + return False + elif among_var == 3: + # (, line 65 + # <-, line 65 + if not self.slice_from(u"a"): + return False + elif among_var == 4: + # (, line 66 + # <-, line 66 + if not self.slice_from(u"o"): + return False + elif among_var == 5: + # (, line 67 + # <-, line 67 + if not self.slice_from(u"u"): + return False + elif among_var == 6: + # (, line 68 + # next, line 68 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_standard_suffix(self): + # (, line 78 + # do, line 79 + v_1 = self.limit - self.cursor + try: + # (, line 79 + # [, line 80 + self.ket = self.cursor + # substring, line 80 + among_var = self.find_among_b(GermanStemmer.a_1, 7) + if among_var == 0: + raise lab0() + # ], line 80 + self.bra = self.cursor + # call R1, line 80 + if not self.r_R1(): + raise lab0() + if among_var == 0: + raise lab0() + elif among_var == 1: + # (, line 82 + # delete, line 82 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 85 + # delete, line 85 + if not self.slice_del(): + return False + + # try, line 86 + v_2 = self.limit - self.cursor + try: + # (, line 86 + # [, line 86 + self.ket = self.cursor + # literal, line 86 + if not self.eq_s_b(1, u"s"): + self.cursor = self.limit - v_2 + raise lab1() + # ], line 86 + self.bra = self.cursor + # literal, line 86 + if not self.eq_s_b(3, u"nis"): + self.cursor = self.limit - v_2 + raise lab1() + # delete, line 86 + if not self.slice_del(): + return False + + except lab1: pass + elif among_var == 3: + # (, line 89 + if not self.in_grouping_b(GermanStemmer.g_s_ending, 98, 116): + raise lab0() + # delete, line 89 + if not self.slice_del(): + return False + + except lab0: pass + self.cursor = self.limit - v_1 + # do, line 93 + v_3 = self.limit - self.cursor + try: + # (, line 93 + # [, line 94 + self.ket = self.cursor + # substring, line 94 + among_var = self.find_among_b(GermanStemmer.a_2, 4) + if among_var == 0: + raise lab2() + # ], line 94 + self.bra = self.cursor + # call R1, line 94 + if not self.r_R1(): + raise lab2() + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 96 + # delete, line 96 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 99 + if not self.in_grouping_b(GermanStemmer.g_st_ending, 98, 116): + raise lab2() + # hop, line 99 + c = self.cursor - 3 + if self.limit_backward > c or c > self.limit: + raise lab2() + self.cursor = c + # delete, line 99 + if not self.slice_del(): + return False + + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 103 + v_4 = self.limit - self.cursor + try: + # (, line 103 + # [, line 104 + self.ket = self.cursor + # substring, line 104 + among_var = self.find_among_b(GermanStemmer.a_4, 8) + if among_var == 0: + raise lab3() + # ], line 104 + self.bra = self.cursor + # call R2, line 104 + if not self.r_R2(): + raise lab3() + if among_var == 0: + raise lab3() + elif among_var == 1: + # (, line 106 + # delete, line 106 + if not self.slice_del(): + return False + + # try, line 107 + v_5 = self.limit - self.cursor + try: + # (, line 107 + # [, line 107 + self.ket = self.cursor + # literal, line 107 + if not self.eq_s_b(2, u"ig"): + self.cursor = self.limit - v_5 + raise lab4() + # ], line 107 + self.bra = self.cursor + # not, line 107 + v_6 = self.limit - self.cursor + try: + # literal, line 107 + if not self.eq_s_b(1, u"e"): + raise lab5() + self.cursor = self.limit - v_5 + raise lab4() + except lab5: pass + self.cursor = self.limit - v_6 + # call R2, line 107 + if not self.r_R2(): + self.cursor = self.limit - v_5 + raise lab4() + # delete, line 107 + if not self.slice_del(): + return False + + except lab4: pass + elif among_var == 2: + # (, line 110 + # not, line 110 + v_7 = self.limit - self.cursor + try: + # literal, line 110 + if not self.eq_s_b(1, u"e"): + raise lab6() + raise lab3() + except lab6: pass + self.cursor = self.limit - v_7 + # delete, line 110 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 113 + # delete, line 113 + if not self.slice_del(): + return False + + # try, line 114 + v_8 = self.limit - self.cursor + try: + # (, line 114 + # [, line 115 + self.ket = self.cursor + # or, line 115 + try: + v_9 = self.limit - self.cursor + try: + # literal, line 115 + if not self.eq_s_b(2, u"er"): + raise lab9() + raise lab8() + except lab9: pass + self.cursor = self.limit - v_9 + # literal, line 115 + if not self.eq_s_b(2, u"en"): + self.cursor = self.limit - v_8 + raise lab7() + except lab8: pass + # ], line 115 + self.bra = self.cursor + # call R1, line 115 + if not self.r_R1(): + self.cursor = self.limit - v_8 + raise lab7() + # delete, line 115 + if not self.slice_del(): + return False + + except lab7: pass + elif among_var == 4: + # (, line 119 + # delete, line 119 + if not self.slice_del(): + return False + + # try, line 120 + v_10 = self.limit - self.cursor + try: + # (, line 120 + # [, line 121 + self.ket = self.cursor + # substring, line 121 + among_var = self.find_among_b(GermanStemmer.a_3, 2) + if among_var == 0: + self.cursor = self.limit - v_10 + raise lab10() + # ], line 121 + self.bra = self.cursor + # call R2, line 121 + if not self.r_R2(): + self.cursor = self.limit - v_10 + raise lab10() + if among_var == 0: + self.cursor = self.limit - v_10 + raise lab10() + elif among_var == 1: + # (, line 123 + # delete, line 123 + if not self.slice_del(): + return False + + except lab10: pass + except lab3: pass + self.cursor = self.limit - v_4 + return True + + def _stem(self): + # (, line 133 + # do, line 134 + v_1 = self.cursor + try: + # call prelude, line 134 + if not self.r_prelude(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # do, line 135 + v_2 = self.cursor + try: + # call mark_regions, line 135 + if not self.r_mark_regions(): + raise lab1() + except lab1: pass + self.cursor = v_2 + # backwards, line 136 + self.limit_backward = self.cursor + self.cursor = self.limit + # do, line 137 + v_3 = self.limit - self.cursor + try: + # call standard_suffix, line 137 + if not self.r_standard_suffix(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + self.cursor = self.limit_backward + # do, line 138 + v_4 = self.cursor + try: + # call postlude, line 138 + if not self.r_postlude(): + raise lab3() + except lab3: pass + self.cursor = v_4 + return True + + def equals(self, o): + return isinstance(o, GermanStemmer) + + def hashCode(self): + return hash("GermanStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass diff --git a/contrib/snowballstemmer/hungarian_stemmer.py b/contrib/snowballstemmer/hungarian_stemmer.py new file mode 100644 index 0000000..688e3d5 --- /dev/null +++ b/contrib/snowballstemmer/hungarian_stemmer.py @@ -0,0 +1,1061 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class HungarianStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"cs", -1, -1), + Among(u"dzs", -1, -1), + Among(u"gy", -1, -1), + Among(u"ly", -1, -1), + Among(u"ny", -1, -1), + Among(u"sz", -1, -1), + Among(u"ty", -1, -1), + Among(u"zs", -1, -1) + ] + + a_1 = [ + Among(u"\u00E1", -1, 1), + Among(u"\u00E9", -1, 2) + ] + + a_2 = [ + Among(u"bb", -1, -1), + Among(u"cc", -1, -1), + Among(u"dd", -1, -1), + Among(u"ff", -1, -1), + Among(u"gg", -1, -1), + Among(u"jj", -1, -1), + Among(u"kk", -1, -1), + Among(u"ll", -1, -1), + Among(u"mm", -1, -1), + Among(u"nn", -1, -1), + Among(u"pp", -1, -1), + Among(u"rr", -1, -1), + Among(u"ccs", -1, -1), + Among(u"ss", -1, -1), + Among(u"zzs", -1, -1), + Among(u"tt", -1, -1), + Among(u"vv", -1, -1), + Among(u"ggy", -1, -1), + Among(u"lly", -1, -1), + Among(u"nny", -1, -1), + Among(u"tty", -1, -1), + Among(u"ssz", -1, -1), + Among(u"zz", -1, -1) + ] + + a_3 = [ + Among(u"al", -1, 1), + Among(u"el", -1, 2) + ] + + a_4 = [ + Among(u"ba", -1, -1), + Among(u"ra", -1, -1), + Among(u"be", -1, -1), + Among(u"re", -1, -1), + Among(u"ig", -1, -1), + Among(u"nak", -1, -1), + Among(u"nek", -1, -1), + Among(u"val", -1, -1), + Among(u"vel", -1, -1), + Among(u"ul", -1, -1), + Among(u"n\u00E1l", -1, -1), + Among(u"n\u00E9l", -1, -1), + Among(u"b\u00F3l", -1, -1), + Among(u"r\u00F3l", -1, -1), + Among(u"t\u00F3l", -1, -1), + Among(u"b\u00F5l", -1, -1), + Among(u"r\u00F5l", -1, -1), + Among(u"t\u00F5l", -1, -1), + Among(u"\u00FCl", -1, -1), + Among(u"n", -1, -1), + Among(u"an", 19, -1), + Among(u"ban", 20, -1), + Among(u"en", 19, -1), + Among(u"ben", 22, -1), + Among(u"k\u00E9ppen", 22, -1), + Among(u"on", 19, -1), + Among(u"\u00F6n", 19, -1), + Among(u"k\u00E9pp", -1, -1), + Among(u"kor", -1, -1), + Among(u"t", -1, -1), + Among(u"at", 29, -1), + Among(u"et", 29, -1), + Among(u"k\u00E9nt", 29, -1), + Among(u"ank\u00E9nt", 32, -1), + Among(u"enk\u00E9nt", 32, -1), + Among(u"onk\u00E9nt", 32, -1), + Among(u"ot", 29, -1), + Among(u"\u00E9rt", 29, -1), + Among(u"\u00F6t", 29, -1), + Among(u"hez", -1, -1), + Among(u"hoz", -1, -1), + Among(u"h\u00F6z", -1, -1), + Among(u"v\u00E1", -1, -1), + Among(u"v\u00E9", -1, -1) + ] + + a_5 = [ + Among(u"\u00E1n", -1, 2), + Among(u"\u00E9n", -1, 1), + Among(u"\u00E1nk\u00E9nt", -1, 3) + ] + + a_6 = [ + Among(u"stul", -1, 2), + Among(u"astul", 0, 1), + Among(u"\u00E1stul", 0, 3), + Among(u"st\u00FCl", -1, 2), + Among(u"est\u00FCl", 3, 1), + Among(u"\u00E9st\u00FCl", 3, 4) + ] + + a_7 = [ + Among(u"\u00E1", -1, 1), + Among(u"\u00E9", -1, 2) + ] + + a_8 = [ + Among(u"k", -1, 7), + Among(u"ak", 0, 4), + Among(u"ek", 0, 6), + Among(u"ok", 0, 5), + Among(u"\u00E1k", 0, 1), + Among(u"\u00E9k", 0, 2), + Among(u"\u00F6k", 0, 3) + ] + + a_9 = [ + Among(u"\u00E9i", -1, 7), + Among(u"\u00E1\u00E9i", 0, 6), + Among(u"\u00E9\u00E9i", 0, 5), + Among(u"\u00E9", -1, 9), + Among(u"k\u00E9", 3, 4), + Among(u"ak\u00E9", 4, 1), + Among(u"ek\u00E9", 4, 1), + Among(u"ok\u00E9", 4, 1), + Among(u"\u00E1k\u00E9", 4, 3), + Among(u"\u00E9k\u00E9", 4, 2), + Among(u"\u00F6k\u00E9", 4, 1), + Among(u"\u00E9\u00E9", 3, 8) + ] + + a_10 = [ + Among(u"a", -1, 18), + Among(u"ja", 0, 17), + Among(u"d", -1, 16), + Among(u"ad", 2, 13), + Among(u"ed", 2, 13), + Among(u"od", 2, 13), + Among(u"\u00E1d", 2, 14), + Among(u"\u00E9d", 2, 15), + Among(u"\u00F6d", 2, 13), + Among(u"e", -1, 18), + Among(u"je", 9, 17), + Among(u"nk", -1, 4), + Among(u"unk", 11, 1), + Among(u"\u00E1nk", 11, 2), + Among(u"\u00E9nk", 11, 3), + Among(u"\u00FCnk", 11, 1), + Among(u"uk", -1, 8), + Among(u"juk", 16, 7), + Among(u"\u00E1juk", 17, 5), + Among(u"\u00FCk", -1, 8), + Among(u"j\u00FCk", 19, 7), + Among(u"\u00E9j\u00FCk", 20, 6), + Among(u"m", -1, 12), + Among(u"am", 22, 9), + Among(u"em", 22, 9), + Among(u"om", 22, 9), + Among(u"\u00E1m", 22, 10), + Among(u"\u00E9m", 22, 11), + Among(u"o", -1, 18), + Among(u"\u00E1", -1, 19), + Among(u"\u00E9", -1, 20) + ] + + a_11 = [ + Among(u"id", -1, 10), + Among(u"aid", 0, 9), + Among(u"jaid", 1, 6), + Among(u"eid", 0, 9), + Among(u"jeid", 3, 6), + Among(u"\u00E1id", 0, 7), + Among(u"\u00E9id", 0, 8), + Among(u"i", -1, 15), + Among(u"ai", 7, 14), + Among(u"jai", 8, 11), + Among(u"ei", 7, 14), + Among(u"jei", 10, 11), + Among(u"\u00E1i", 7, 12), + Among(u"\u00E9i", 7, 13), + Among(u"itek", -1, 24), + Among(u"eitek", 14, 21), + Among(u"jeitek", 15, 20), + Among(u"\u00E9itek", 14, 23), + Among(u"ik", -1, 29), + Among(u"aik", 18, 26), + Among(u"jaik", 19, 25), + Among(u"eik", 18, 26), + Among(u"jeik", 21, 25), + Among(u"\u00E1ik", 18, 27), + Among(u"\u00E9ik", 18, 28), + Among(u"ink", -1, 20), + Among(u"aink", 25, 17), + Among(u"jaink", 26, 16), + Among(u"eink", 25, 17), + Among(u"jeink", 28, 16), + Among(u"\u00E1ink", 25, 18), + Among(u"\u00E9ink", 25, 19), + Among(u"aitok", -1, 21), + Among(u"jaitok", 32, 20), + Among(u"\u00E1itok", -1, 22), + Among(u"im", -1, 5), + Among(u"aim", 35, 4), + Among(u"jaim", 36, 1), + Among(u"eim", 35, 4), + Among(u"jeim", 38, 1), + Among(u"\u00E1im", 35, 2), + Among(u"\u00E9im", 35, 3) + ] + + g_v = [17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 52, 14] + + I_p1 = 0 + + def copy_from(self, other): + self.I_p1 = other.I_p1 + super.copy_from(other) + + + def r_mark_regions(self): + # (, line 44 + self.I_p1 = self.limit; + # or, line 51 + try: + v_1 = self.cursor + try: + # (, line 48 + if not self.in_grouping(HungarianStemmer.g_v, 97, 252): + raise lab1() + # goto, line 48 + try: + while True: + v_2 = self.cursor + try: + if not self.out_grouping(HungarianStemmer.g_v, 97, 252): + raise lab3() + self.cursor = v_2 + raise lab2() + except lab3: pass + self.cursor = v_2 + if self.cursor >= self.limit: + raise lab1() + self.cursor += 1 + except lab2: pass + # or, line 49 + try: + v_3 = self.cursor + try: + # among, line 49 + if self.find_among(HungarianStemmer.a_0, 8) == 0: + raise lab5() + raise lab4() + except lab5: pass + self.cursor = v_3 + # next, line 49 + if self.cursor >= self.limit: + raise lab1() + self.cursor += 1 + except lab4: pass + # setmark p1, line 50 + self.I_p1 = self.cursor + raise lab0() + except lab1: pass + self.cursor = v_1 + # (, line 53 + if not self.out_grouping(HungarianStemmer.g_v, 97, 252): + return False + # gopast, line 53 + try: + while True: + try: + if not self.in_grouping(HungarianStemmer.g_v, 97, 252): + raise lab7() + raise lab6() + except lab7: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab6: pass + # setmark p1, line 53 + self.I_p1 = self.cursor + except lab0: pass + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_v_ending(self): + # (, line 60 + # [, line 61 + self.ket = self.cursor + # substring, line 61 + among_var = self.find_among_b(HungarianStemmer.a_1, 2) + if among_var == 0: + return False + # ], line 61 + self.bra = self.cursor + # call R1, line 61 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 62 + # <-, line 62 + if not self.slice_from(u"a"): + return False + elif among_var == 2: + # (, line 63 + # <-, line 63 + if not self.slice_from(u"e"): + return False + return True + + def r_double(self): + # (, line 67 + # test, line 68 + v_1 = self.limit - self.cursor + # among, line 68 + if self.find_among_b(HungarianStemmer.a_2, 23) == 0: + return False + self.cursor = self.limit - v_1 + return True + + def r_undouble(self): + # (, line 72 + # next, line 73 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # [, line 73 + self.ket = self.cursor + # hop, line 73 + c = self.cursor - 1 + if self.limit_backward > c or c > self.limit: + return False + self.cursor = c + # ], line 73 + self.bra = self.cursor + # delete, line 73 + if not self.slice_del(): + return False + + return True + + def r_instrum(self): + # (, line 76 + # [, line 77 + self.ket = self.cursor + # substring, line 77 + among_var = self.find_among_b(HungarianStemmer.a_3, 2) + if among_var == 0: + return False + # ], line 77 + self.bra = self.cursor + # call R1, line 77 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 78 + # call double, line 78 + if not self.r_double(): + return False + elif among_var == 2: + # (, line 79 + # call double, line 79 + if not self.r_double(): + return False + # delete, line 81 + if not self.slice_del(): + return False + + # call undouble, line 82 + if not self.r_undouble(): + return False + return True + + def r_case(self): + # (, line 86 + # [, line 87 + self.ket = self.cursor + # substring, line 87 + if self.find_among_b(HungarianStemmer.a_4, 44) == 0: + return False + # ], line 87 + self.bra = self.cursor + # call R1, line 87 + if not self.r_R1(): + return False + # delete, line 111 + if not self.slice_del(): + return False + + # call v_ending, line 112 + if not self.r_v_ending(): + return False + return True + + def r_case_special(self): + # (, line 115 + # [, line 116 + self.ket = self.cursor + # substring, line 116 + among_var = self.find_among_b(HungarianStemmer.a_5, 3) + if among_var == 0: + return False + # ], line 116 + self.bra = self.cursor + # call R1, line 116 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 117 + # <-, line 117 + if not self.slice_from(u"e"): + return False + elif among_var == 2: + # (, line 118 + # <-, line 118 + if not self.slice_from(u"a"): + return False + elif among_var == 3: + # (, line 119 + # <-, line 119 + if not self.slice_from(u"a"): + return False + return True + + def r_case_other(self): + # (, line 123 + # [, line 124 + self.ket = self.cursor + # substring, line 124 + among_var = self.find_among_b(HungarianStemmer.a_6, 6) + if among_var == 0: + return False + # ], line 124 + self.bra = self.cursor + # call R1, line 124 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 125 + # delete, line 125 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 126 + # delete, line 126 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 127 + # <-, line 127 + if not self.slice_from(u"a"): + return False + elif among_var == 4: + # (, line 128 + # <-, line 128 + if not self.slice_from(u"e"): + return False + return True + + def r_factive(self): + # (, line 132 + # [, line 133 + self.ket = self.cursor + # substring, line 133 + among_var = self.find_among_b(HungarianStemmer.a_7, 2) + if among_var == 0: + return False + # ], line 133 + self.bra = self.cursor + # call R1, line 133 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 134 + # call double, line 134 + if not self.r_double(): + return False + elif among_var == 2: + # (, line 135 + # call double, line 135 + if not self.r_double(): + return False + # delete, line 137 + if not self.slice_del(): + return False + + # call undouble, line 138 + if not self.r_undouble(): + return False + return True + + def r_plural(self): + # (, line 141 + # [, line 142 + self.ket = self.cursor + # substring, line 142 + among_var = self.find_among_b(HungarianStemmer.a_8, 7) + if among_var == 0: + return False + # ], line 142 + self.bra = self.cursor + # call R1, line 142 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 143 + # <-, line 143 + if not self.slice_from(u"a"): + return False + elif among_var == 2: + # (, line 144 + # <-, line 144 + if not self.slice_from(u"e"): + return False + elif among_var == 3: + # (, line 145 + # delete, line 145 + if not self.slice_del(): + return False + + elif among_var == 4: + # (, line 146 + # delete, line 146 + if not self.slice_del(): + return False + + elif among_var == 5: + # (, line 147 + # delete, line 147 + if not self.slice_del(): + return False + + elif among_var == 6: + # (, line 148 + # delete, line 148 + if not self.slice_del(): + return False + + elif among_var == 7: + # (, line 149 + # delete, line 149 + if not self.slice_del(): + return False + + return True + + def r_owned(self): + # (, line 153 + # [, line 154 + self.ket = self.cursor + # substring, line 154 + among_var = self.find_among_b(HungarianStemmer.a_9, 12) + if among_var == 0: + return False + # ], line 154 + self.bra = self.cursor + # call R1, line 154 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 155 + # delete, line 155 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 156 + # <-, line 156 + if not self.slice_from(u"e"): + return False + elif among_var == 3: + # (, line 157 + # <-, line 157 + if not self.slice_from(u"a"): + return False + elif among_var == 4: + # (, line 158 + # delete, line 158 + if not self.slice_del(): + return False + + elif among_var == 5: + # (, line 159 + # <-, line 159 + if not self.slice_from(u"e"): + return False + elif among_var == 6: + # (, line 160 + # <-, line 160 + if not self.slice_from(u"a"): + return False + elif among_var == 7: + # (, line 161 + # delete, line 161 + if not self.slice_del(): + return False + + elif among_var == 8: + # (, line 162 + # <-, line 162 + if not self.slice_from(u"e"): + return False + elif among_var == 9: + # (, line 163 + # delete, line 163 + if not self.slice_del(): + return False + + return True + + def r_sing_owner(self): + # (, line 167 + # [, line 168 + self.ket = self.cursor + # substring, line 168 + among_var = self.find_among_b(HungarianStemmer.a_10, 31) + if among_var == 0: + return False + # ], line 168 + self.bra = self.cursor + # call R1, line 168 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 169 + # delete, line 169 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 170 + # <-, line 170 + if not self.slice_from(u"a"): + return False + elif among_var == 3: + # (, line 171 + # <-, line 171 + if not self.slice_from(u"e"): + return False + elif among_var == 4: + # (, line 172 + # delete, line 172 + if not self.slice_del(): + return False + + elif among_var == 5: + # (, line 173 + # <-, line 173 + if not self.slice_from(u"a"): + return False + elif among_var == 6: + # (, line 174 + # <-, line 174 + if not self.slice_from(u"e"): + return False + elif among_var == 7: + # (, line 175 + # delete, line 175 + if not self.slice_del(): + return False + + elif among_var == 8: + # (, line 176 + # delete, line 176 + if not self.slice_del(): + return False + + elif among_var == 9: + # (, line 177 + # delete, line 177 + if not self.slice_del(): + return False + + elif among_var == 10: + # (, line 178 + # <-, line 178 + if not self.slice_from(u"a"): + return False + elif among_var == 11: + # (, line 179 + # <-, line 179 + if not self.slice_from(u"e"): + return False + elif among_var == 12: + # (, line 180 + # delete, line 180 + if not self.slice_del(): + return False + + elif among_var == 13: + # (, line 181 + # delete, line 181 + if not self.slice_del(): + return False + + elif among_var == 14: + # (, line 182 + # <-, line 182 + if not self.slice_from(u"a"): + return False + elif among_var == 15: + # (, line 183 + # <-, line 183 + if not self.slice_from(u"e"): + return False + elif among_var == 16: + # (, line 184 + # delete, line 184 + if not self.slice_del(): + return False + + elif among_var == 17: + # (, line 185 + # delete, line 185 + if not self.slice_del(): + return False + + elif among_var == 18: + # (, line 186 + # delete, line 186 + if not self.slice_del(): + return False + + elif among_var == 19: + # (, line 187 + # <-, line 187 + if not self.slice_from(u"a"): + return False + elif among_var == 20: + # (, line 188 + # <-, line 188 + if not self.slice_from(u"e"): + return False + return True + + def r_plur_owner(self): + # (, line 192 + # [, line 193 + self.ket = self.cursor + # substring, line 193 + among_var = self.find_among_b(HungarianStemmer.a_11, 42) + if among_var == 0: + return False + # ], line 193 + self.bra = self.cursor + # call R1, line 193 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 194 + # delete, line 194 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 195 + # <-, line 195 + if not self.slice_from(u"a"): + return False + elif among_var == 3: + # (, line 196 + # <-, line 196 + if not self.slice_from(u"e"): + return False + elif among_var == 4: + # (, line 197 + # delete, line 197 + if not self.slice_del(): + return False + + elif among_var == 5: + # (, line 198 + # delete, line 198 + if not self.slice_del(): + return False + + elif among_var == 6: + # (, line 199 + # delete, line 199 + if not self.slice_del(): + return False + + elif among_var == 7: + # (, line 200 + # <-, line 200 + if not self.slice_from(u"a"): + return False + elif among_var == 8: + # (, line 201 + # <-, line 201 + if not self.slice_from(u"e"): + return False + elif among_var == 9: + # (, line 202 + # delete, line 202 + if not self.slice_del(): + return False + + elif among_var == 10: + # (, line 203 + # delete, line 203 + if not self.slice_del(): + return False + + elif among_var == 11: + # (, line 204 + # delete, line 204 + if not self.slice_del(): + return False + + elif among_var == 12: + # (, line 205 + # <-, line 205 + if not self.slice_from(u"a"): + return False + elif among_var == 13: + # (, line 206 + # <-, line 206 + if not self.slice_from(u"e"): + return False + elif among_var == 14: + # (, line 207 + # delete, line 207 + if not self.slice_del(): + return False + + elif among_var == 15: + # (, line 208 + # delete, line 208 + if not self.slice_del(): + return False + + elif among_var == 16: + # (, line 209 + # delete, line 209 + if not self.slice_del(): + return False + + elif among_var == 17: + # (, line 210 + # delete, line 210 + if not self.slice_del(): + return False + + elif among_var == 18: + # (, line 211 + # <-, line 211 + if not self.slice_from(u"a"): + return False + elif among_var == 19: + # (, line 212 + # <-, line 212 + if not self.slice_from(u"e"): + return False + elif among_var == 20: + # (, line 214 + # delete, line 214 + if not self.slice_del(): + return False + + elif among_var == 21: + # (, line 215 + # delete, line 215 + if not self.slice_del(): + return False + + elif among_var == 22: + # (, line 216 + # <-, line 216 + if not self.slice_from(u"a"): + return False + elif among_var == 23: + # (, line 217 + # <-, line 217 + if not self.slice_from(u"e"): + return False + elif among_var == 24: + # (, line 218 + # delete, line 218 + if not self.slice_del(): + return False + + elif among_var == 25: + # (, line 219 + # delete, line 219 + if not self.slice_del(): + return False + + elif among_var == 26: + # (, line 220 + # delete, line 220 + if not self.slice_del(): + return False + + elif among_var == 27: + # (, line 221 + # <-, line 221 + if not self.slice_from(u"a"): + return False + elif among_var == 28: + # (, line 222 + # <-, line 222 + if not self.slice_from(u"e"): + return False + elif among_var == 29: + # (, line 223 + # delete, line 223 + if not self.slice_del(): + return False + + return True + + def _stem(self): + # (, line 228 + # do, line 229 + v_1 = self.cursor + try: + # call mark_regions, line 229 + if not self.r_mark_regions(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # backwards, line 230 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 230 + # do, line 231 + v_2 = self.limit - self.cursor + try: + # call instrum, line 231 + if not self.r_instrum(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 232 + v_3 = self.limit - self.cursor + try: + # call case, line 232 + if not self.r_case(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 233 + v_4 = self.limit - self.cursor + try: + # call case_special, line 233 + if not self.r_case_special(): + raise lab3() + except lab3: pass + self.cursor = self.limit - v_4 + # do, line 234 + v_5 = self.limit - self.cursor + try: + # call case_other, line 234 + if not self.r_case_other(): + raise lab4() + except lab4: pass + self.cursor = self.limit - v_5 + # do, line 235 + v_6 = self.limit - self.cursor + try: + # call factive, line 235 + if not self.r_factive(): + raise lab5() + except lab5: pass + self.cursor = self.limit - v_6 + # do, line 236 + v_7 = self.limit - self.cursor + try: + # call owned, line 236 + if not self.r_owned(): + raise lab6() + except lab6: pass + self.cursor = self.limit - v_7 + # do, line 237 + v_8 = self.limit - self.cursor + try: + # call sing_owner, line 237 + if not self.r_sing_owner(): + raise lab7() + except lab7: pass + self.cursor = self.limit - v_8 + # do, line 238 + v_9 = self.limit - self.cursor + try: + # call plur_owner, line 238 + if not self.r_plur_owner(): + raise lab8() + except lab8: pass + self.cursor = self.limit - v_9 + # do, line 239 + v_10 = self.limit - self.cursor + try: + # call plural, line 239 + if not self.r_plural(): + raise lab9() + except lab9: pass + self.cursor = self.limit - v_10 + self.cursor = self.limit_backward + return True + + def equals(self, o): + return isinstance(o, HungarianStemmer) + + def hashCode(self): + return hash("HungarianStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass diff --git a/contrib/snowballstemmer/italian_stemmer.py b/contrib/snowballstemmer/italian_stemmer.py new file mode 100644 index 0000000..02ebed9 --- /dev/null +++ b/contrib/snowballstemmer/italian_stemmer.py @@ -0,0 +1,1033 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class ItalianStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"", -1, 7), + Among(u"qu", 0, 6), + Among(u"\u00E1", 0, 1), + Among(u"\u00E9", 0, 2), + Among(u"\u00ED", 0, 3), + Among(u"\u00F3", 0, 4), + Among(u"\u00FA", 0, 5) + ] + + a_1 = [ + Among(u"", -1, 3), + Among(u"I", 0, 1), + Among(u"U", 0, 2) + ] + + a_2 = [ + Among(u"la", -1, -1), + Among(u"cela", 0, -1), + Among(u"gliela", 0, -1), + Among(u"mela", 0, -1), + Among(u"tela", 0, -1), + Among(u"vela", 0, -1), + Among(u"le", -1, -1), + Among(u"cele", 6, -1), + Among(u"gliele", 6, -1), + Among(u"mele", 6, -1), + Among(u"tele", 6, -1), + Among(u"vele", 6, -1), + Among(u"ne", -1, -1), + Among(u"cene", 12, -1), + Among(u"gliene", 12, -1), + Among(u"mene", 12, -1), + Among(u"sene", 12, -1), + Among(u"tene", 12, -1), + Among(u"vene", 12, -1), + Among(u"ci", -1, -1), + Among(u"li", -1, -1), + Among(u"celi", 20, -1), + Among(u"glieli", 20, -1), + Among(u"meli", 20, -1), + Among(u"teli", 20, -1), + Among(u"veli", 20, -1), + Among(u"gli", 20, -1), + Among(u"mi", -1, -1), + Among(u"si", -1, -1), + Among(u"ti", -1, -1), + Among(u"vi", -1, -1), + Among(u"lo", -1, -1), + Among(u"celo", 31, -1), + Among(u"glielo", 31, -1), + Among(u"melo", 31, -1), + Among(u"telo", 31, -1), + Among(u"velo", 31, -1) + ] + + a_3 = [ + Among(u"ando", -1, 1), + Among(u"endo", -1, 1), + Among(u"ar", -1, 2), + Among(u"er", -1, 2), + Among(u"ir", -1, 2) + ] + + a_4 = [ + Among(u"ic", -1, -1), + Among(u"abil", -1, -1), + Among(u"os", -1, -1), + Among(u"iv", -1, 1) + ] + + a_5 = [ + Among(u"ic", -1, 1), + Among(u"abil", -1, 1), + Among(u"iv", -1, 1) + ] + + a_6 = [ + Among(u"ica", -1, 1), + Among(u"logia", -1, 3), + Among(u"osa", -1, 1), + Among(u"ista", -1, 1), + Among(u"iva", -1, 9), + Among(u"anza", -1, 1), + Among(u"enza", -1, 5), + Among(u"ice", -1, 1), + Among(u"atrice", 7, 1), + Among(u"iche", -1, 1), + Among(u"logie", -1, 3), + Among(u"abile", -1, 1), + Among(u"ibile", -1, 1), + Among(u"usione", -1, 4), + Among(u"azione", -1, 2), + Among(u"uzione", -1, 4), + Among(u"atore", -1, 2), + Among(u"ose", -1, 1), + Among(u"ante", -1, 1), + Among(u"mente", -1, 1), + Among(u"amente", 19, 7), + Among(u"iste", -1, 1), + Among(u"ive", -1, 9), + Among(u"anze", -1, 1), + Among(u"enze", -1, 5), + Among(u"ici", -1, 1), + Among(u"atrici", 25, 1), + Among(u"ichi", -1, 1), + Among(u"abili", -1, 1), + Among(u"ibili", -1, 1), + Among(u"ismi", -1, 1), + Among(u"usioni", -1, 4), + Among(u"azioni", -1, 2), + Among(u"uzioni", -1, 4), + Among(u"atori", -1, 2), + Among(u"osi", -1, 1), + Among(u"anti", -1, 1), + Among(u"amenti", -1, 6), + Among(u"imenti", -1, 6), + Among(u"isti", -1, 1), + Among(u"ivi", -1, 9), + Among(u"ico", -1, 1), + Among(u"ismo", -1, 1), + Among(u"oso", -1, 1), + Among(u"amento", -1, 6), + Among(u"imento", -1, 6), + Among(u"ivo", -1, 9), + Among(u"it\u00E0", -1, 8), + Among(u"ist\u00E0", -1, 1), + Among(u"ist\u00E8", -1, 1), + Among(u"ist\u00EC", -1, 1) + ] + + a_7 = [ + Among(u"isca", -1, 1), + Among(u"enda", -1, 1), + Among(u"ata", -1, 1), + Among(u"ita", -1, 1), + Among(u"uta", -1, 1), + Among(u"ava", -1, 1), + Among(u"eva", -1, 1), + Among(u"iva", -1, 1), + Among(u"erebbe", -1, 1), + Among(u"irebbe", -1, 1), + Among(u"isce", -1, 1), + Among(u"ende", -1, 1), + Among(u"are", -1, 1), + Among(u"ere", -1, 1), + Among(u"ire", -1, 1), + Among(u"asse", -1, 1), + Among(u"ate", -1, 1), + Among(u"avate", 16, 1), + Among(u"evate", 16, 1), + Among(u"ivate", 16, 1), + Among(u"ete", -1, 1), + Among(u"erete", 20, 1), + Among(u"irete", 20, 1), + Among(u"ite", -1, 1), + Among(u"ereste", -1, 1), + Among(u"ireste", -1, 1), + Among(u"ute", -1, 1), + Among(u"erai", -1, 1), + Among(u"irai", -1, 1), + Among(u"isci", -1, 1), + Among(u"endi", -1, 1), + Among(u"erei", -1, 1), + Among(u"irei", -1, 1), + Among(u"assi", -1, 1), + Among(u"ati", -1, 1), + Among(u"iti", -1, 1), + Among(u"eresti", -1, 1), + Among(u"iresti", -1, 1), + Among(u"uti", -1, 1), + Among(u"avi", -1, 1), + Among(u"evi", -1, 1), + Among(u"ivi", -1, 1), + Among(u"isco", -1, 1), + Among(u"ando", -1, 1), + Among(u"endo", -1, 1), + Among(u"Yamo", -1, 1), + Among(u"iamo", -1, 1), + Among(u"avamo", -1, 1), + Among(u"evamo", -1, 1), + Among(u"ivamo", -1, 1), + Among(u"eremo", -1, 1), + Among(u"iremo", -1, 1), + Among(u"assimo", -1, 1), + Among(u"ammo", -1, 1), + Among(u"emmo", -1, 1), + Among(u"eremmo", 54, 1), + Among(u"iremmo", 54, 1), + Among(u"immo", -1, 1), + Among(u"ano", -1, 1), + Among(u"iscano", 58, 1), + Among(u"avano", 58, 1), + Among(u"evano", 58, 1), + Among(u"ivano", 58, 1), + Among(u"eranno", -1, 1), + Among(u"iranno", -1, 1), + Among(u"ono", -1, 1), + Among(u"iscono", 65, 1), + Among(u"arono", 65, 1), + Among(u"erono", 65, 1), + Among(u"irono", 65, 1), + Among(u"erebbero", -1, 1), + Among(u"irebbero", -1, 1), + Among(u"assero", -1, 1), + Among(u"essero", -1, 1), + Among(u"issero", -1, 1), + Among(u"ato", -1, 1), + Among(u"ito", -1, 1), + Among(u"uto", -1, 1), + Among(u"avo", -1, 1), + Among(u"evo", -1, 1), + Among(u"ivo", -1, 1), + Among(u"ar", -1, 1), + Among(u"ir", -1, 1), + Among(u"er\u00E0", -1, 1), + Among(u"ir\u00E0", -1, 1), + Among(u"er\u00F2", -1, 1), + Among(u"ir\u00F2", -1, 1) + ] + + g_v = [17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 8, 2, 1] + + g_AEIO = [17, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 8, 2] + + g_CG = [17] + + I_p2 = 0 + I_p1 = 0 + I_pV = 0 + + def copy_from(self, other): + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + self.I_pV = other.I_pV + super.copy_from(other) + + + def r_prelude(self): + # (, line 34 + # test, line 35 + v_1 = self.cursor + # repeat, line 35 + try: + while True: + try: + v_2 = self.cursor + try: + # (, line 35 + # [, line 36 + self.bra = self.cursor + # substring, line 36 + among_var = self.find_among(ItalianStemmer.a_0, 7) + if among_var == 0: + raise lab2() + # ], line 36 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 37 + # <-, line 37 + if not self.slice_from(u"\u00E0"): + return False + elif among_var == 2: + # (, line 38 + # <-, line 38 + if not self.slice_from(u"\u00E8"): + return False + elif among_var == 3: + # (, line 39 + # <-, line 39 + if not self.slice_from(u"\u00EC"): + return False + elif among_var == 4: + # (, line 40 + # <-, line 40 + if not self.slice_from(u"\u00F2"): + return False + elif among_var == 5: + # (, line 41 + # <-, line 41 + if not self.slice_from(u"\u00F9"): + return False + elif among_var == 6: + # (, line 42 + # <-, line 42 + if not self.slice_from(u"qU"): + return False + elif among_var == 7: + # (, line 43 + # next, line 43 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_2 + raise lab0() + except lab1: pass + except lab0: pass + self.cursor = v_1 + # repeat, line 46 + try: + while True: + try: + v_3 = self.cursor + try: + # goto, line 46 + try: + while True: + v_4 = self.cursor + try: + # (, line 46 + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab7() + # [, line 47 + self.bra = self.cursor + # or, line 47 + try: + v_5 = self.cursor + try: + # (, line 47 + # literal, line 47 + if not self.eq_s(1, u"u"): + raise lab9() + # ], line 47 + self.ket = self.cursor + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab9() + # <-, line 47 + if not self.slice_from(u"U"): + return False + raise lab8() + except lab9: pass + self.cursor = v_5 + # (, line 48 + # literal, line 48 + if not self.eq_s(1, u"i"): + raise lab7() + # ], line 48 + self.ket = self.cursor + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab7() + # <-, line 48 + if not self.slice_from(u"I"): + return False + except lab8: pass + self.cursor = v_4 + raise lab6() + except lab7: pass + self.cursor = v_4 + if self.cursor >= self.limit: + raise lab5() + self.cursor += 1 + except lab6: pass + raise lab4() + except lab5: pass + self.cursor = v_3 + raise lab3() + except lab4: pass + except lab3: pass + return True + + def r_mark_regions(self): + # (, line 52 + self.I_pV = self.limit; + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # do, line 58 + v_1 = self.cursor + try: + # (, line 58 + # or, line 60 + try: + v_2 = self.cursor + try: + # (, line 59 + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab2() + # or, line 59 + try: + v_3 = self.cursor + try: + # (, line 59 + if not self.out_grouping(ItalianStemmer.g_v, 97, 249): + raise lab4() + # gopast, line 59 + try: + while True: + try: + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + raise lab4() + self.cursor += 1 + except lab5: pass + raise lab3() + except lab4: pass + self.cursor = v_3 + # (, line 59 + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab2() + # gopast, line 59 + try: + while True: + try: + if not self.out_grouping(ItalianStemmer.g_v, 97, 249): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab7: pass + except lab3: pass + raise lab1() + except lab2: pass + self.cursor = v_2 + # (, line 61 + if not self.out_grouping(ItalianStemmer.g_v, 97, 249): + raise lab0() + # or, line 61 + try: + v_6 = self.cursor + try: + # (, line 61 + if not self.out_grouping(ItalianStemmer.g_v, 97, 249): + raise lab10() + # gopast, line 61 + try: + while True: + try: + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab12() + raise lab11() + except lab12: pass + if self.cursor >= self.limit: + raise lab10() + self.cursor += 1 + except lab11: pass + raise lab9() + except lab10: pass + self.cursor = v_6 + # (, line 61 + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab0() + # next, line 61 + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab9: pass + except lab1: pass + # setmark pV, line 62 + self.I_pV = self.cursor + except lab0: pass + self.cursor = v_1 + # do, line 64 + v_8 = self.cursor + try: + # (, line 64 + # gopast, line 65 + try: + while True: + try: + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab15() + raise lab14() + except lab15: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab14: pass + # gopast, line 65 + try: + while True: + try: + if not self.out_grouping(ItalianStemmer.g_v, 97, 249): + raise lab17() + raise lab16() + except lab17: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab16: pass + # setmark p1, line 65 + self.I_p1 = self.cursor + # gopast, line 66 + try: + while True: + try: + if not self.in_grouping(ItalianStemmer.g_v, 97, 249): + raise lab19() + raise lab18() + except lab19: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab18: pass + # gopast, line 66 + try: + while True: + try: + if not self.out_grouping(ItalianStemmer.g_v, 97, 249): + raise lab21() + raise lab20() + except lab21: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab20: pass + # setmark p2, line 66 + self.I_p2 = self.cursor + except lab13: pass + self.cursor = v_8 + return True + + def r_postlude(self): + # repeat, line 70 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 70 + # [, line 72 + self.bra = self.cursor + # substring, line 72 + among_var = self.find_among(ItalianStemmer.a_1, 3) + if among_var == 0: + raise lab2() + # ], line 72 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 73 + # <-, line 73 + if not self.slice_from(u"i"): + return False + elif among_var == 2: + # (, line 74 + # <-, line 74 + if not self.slice_from(u"u"): + return False + elif among_var == 3: + # (, line 75 + # next, line 75 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_RV(self): + if not self.I_pV <= self.cursor: + return False + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_attached_pronoun(self): + # (, line 86 + # [, line 87 + self.ket = self.cursor + # substring, line 87 + if self.find_among_b(ItalianStemmer.a_2, 37) == 0: + return False + # ], line 87 + self.bra = self.cursor + # among, line 97 + among_var = self.find_among_b(ItalianStemmer.a_3, 5) + if among_var == 0: + return False + # (, line 97 + # call RV, line 97 + if not self.r_RV(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 98 + # delete, line 98 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 99 + # <-, line 99 + if not self.slice_from(u"e"): + return False + return True + + def r_standard_suffix(self): + # (, line 103 + # [, line 104 + self.ket = self.cursor + # substring, line 104 + among_var = self.find_among_b(ItalianStemmer.a_6, 51) + if among_var == 0: + return False + # ], line 104 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 111 + # call R2, line 111 + if not self.r_R2(): + return False + # delete, line 111 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 113 + # call R2, line 113 + if not self.r_R2(): + return False + # delete, line 113 + if not self.slice_del(): + return False + + # try, line 114 + v_1 = self.limit - self.cursor + try: + # (, line 114 + # [, line 114 + self.ket = self.cursor + # literal, line 114 + if not self.eq_s_b(2, u"ic"): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 114 + self.bra = self.cursor + # call R2, line 114 + if not self.r_R2(): + self.cursor = self.limit - v_1 + raise lab0() + # delete, line 114 + if not self.slice_del(): + return False + + except lab0: pass + elif among_var == 3: + # (, line 117 + # call R2, line 117 + if not self.r_R2(): + return False + # <-, line 117 + if not self.slice_from(u"log"): + return False + elif among_var == 4: + # (, line 119 + # call R2, line 119 + if not self.r_R2(): + return False + # <-, line 119 + if not self.slice_from(u"u"): + return False + elif among_var == 5: + # (, line 121 + # call R2, line 121 + if not self.r_R2(): + return False + # <-, line 121 + if not self.slice_from(u"ente"): + return False + elif among_var == 6: + # (, line 123 + # call RV, line 123 + if not self.r_RV(): + return False + # delete, line 123 + if not self.slice_del(): + return False + + elif among_var == 7: + # (, line 124 + # call R1, line 125 + if not self.r_R1(): + return False + # delete, line 125 + if not self.slice_del(): + return False + + # try, line 126 + v_2 = self.limit - self.cursor + try: + # (, line 126 + # [, line 127 + self.ket = self.cursor + # substring, line 127 + among_var = self.find_among_b(ItalianStemmer.a_4, 4) + if among_var == 0: + self.cursor = self.limit - v_2 + raise lab1() + # ], line 127 + self.bra = self.cursor + # call R2, line 127 + if not self.r_R2(): + self.cursor = self.limit - v_2 + raise lab1() + # delete, line 127 + if not self.slice_del(): + return False + + if among_var == 0: + self.cursor = self.limit - v_2 + raise lab1() + elif among_var == 1: + # (, line 128 + # [, line 128 + self.ket = self.cursor + # literal, line 128 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_2 + raise lab1() + # ], line 128 + self.bra = self.cursor + # call R2, line 128 + if not self.r_R2(): + self.cursor = self.limit - v_2 + raise lab1() + # delete, line 128 + if not self.slice_del(): + return False + + except lab1: pass + elif among_var == 8: + # (, line 133 + # call R2, line 134 + if not self.r_R2(): + return False + # delete, line 134 + if not self.slice_del(): + return False + + # try, line 135 + v_3 = self.limit - self.cursor + try: + # (, line 135 + # [, line 136 + self.ket = self.cursor + # substring, line 136 + among_var = self.find_among_b(ItalianStemmer.a_5, 3) + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab2() + # ], line 136 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab2() + elif among_var == 1: + # (, line 137 + # call R2, line 137 + if not self.r_R2(): + self.cursor = self.limit - v_3 + raise lab2() + # delete, line 137 + if not self.slice_del(): + return False + + except lab2: pass + elif among_var == 9: + # (, line 141 + # call R2, line 142 + if not self.r_R2(): + return False + # delete, line 142 + if not self.slice_del(): + return False + + # try, line 143 + v_4 = self.limit - self.cursor + try: + # (, line 143 + # [, line 143 + self.ket = self.cursor + # literal, line 143 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_4 + raise lab3() + # ], line 143 + self.bra = self.cursor + # call R2, line 143 + if not self.r_R2(): + self.cursor = self.limit - v_4 + raise lab3() + # delete, line 143 + if not self.slice_del(): + return False + + # [, line 143 + self.ket = self.cursor + # literal, line 143 + if not self.eq_s_b(2, u"ic"): + self.cursor = self.limit - v_4 + raise lab3() + # ], line 143 + self.bra = self.cursor + # call R2, line 143 + if not self.r_R2(): + self.cursor = self.limit - v_4 + raise lab3() + # delete, line 143 + if not self.slice_del(): + return False + + except lab3: pass + return True + + def r_verb_suffix(self): + # setlimit, line 148 + v_1 = self.limit - self.cursor + # tomark, line 148 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 148 + # [, line 149 + self.ket = self.cursor + # substring, line 149 + among_var = self.find_among_b(ItalianStemmer.a_7, 87) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 149 + self.bra = self.cursor + if among_var == 0: + self.limit_backward = v_2 + return False + elif among_var == 1: + # (, line 163 + # delete, line 163 + if not self.slice_del(): + return False + + self.limit_backward = v_2 + return True + + def r_vowel_suffix(self): + # (, line 170 + # try, line 171 + v_1 = self.limit - self.cursor + try: + # (, line 171 + # [, line 172 + self.ket = self.cursor + if not self.in_grouping_b(ItalianStemmer.g_AEIO, 97, 242): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 172 + self.bra = self.cursor + # call RV, line 172 + if not self.r_RV(): + self.cursor = self.limit - v_1 + raise lab0() + # delete, line 172 + if not self.slice_del(): + return False + + # [, line 173 + self.ket = self.cursor + # literal, line 173 + if not self.eq_s_b(1, u"i"): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 173 + self.bra = self.cursor + # call RV, line 173 + if not self.r_RV(): + self.cursor = self.limit - v_1 + raise lab0() + # delete, line 173 + if not self.slice_del(): + return False + + except lab0: pass + # try, line 175 + v_2 = self.limit - self.cursor + try: + # (, line 175 + # [, line 176 + self.ket = self.cursor + # literal, line 176 + if not self.eq_s_b(1, u"h"): + self.cursor = self.limit - v_2 + raise lab1() + # ], line 176 + self.bra = self.cursor + if not self.in_grouping_b(ItalianStemmer.g_CG, 99, 103): + self.cursor = self.limit - v_2 + raise lab1() + # call RV, line 176 + if not self.r_RV(): + self.cursor = self.limit - v_2 + raise lab1() + # delete, line 176 + if not self.slice_del(): + return False + + except lab1: pass + return True + + def _stem(self): + # (, line 181 + # do, line 182 + v_1 = self.cursor + try: + # call prelude, line 182 + if not self.r_prelude(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # do, line 183 + v_2 = self.cursor + try: + # call mark_regions, line 183 + if not self.r_mark_regions(): + raise lab1() + except lab1: pass + self.cursor = v_2 + # backwards, line 184 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 184 + # do, line 185 + v_3 = self.limit - self.cursor + try: + # call attached_pronoun, line 185 + if not self.r_attached_pronoun(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 186 + v_4 = self.limit - self.cursor + try: + # (, line 186 + # or, line 186 + try: + v_5 = self.limit - self.cursor + try: + # call standard_suffix, line 186 + if not self.r_standard_suffix(): + raise lab5() + raise lab4() + except lab5: pass + self.cursor = self.limit - v_5 + # call verb_suffix, line 186 + if not self.r_verb_suffix(): + raise lab3() + except lab4: pass + except lab3: pass + self.cursor = self.limit - v_4 + # do, line 187 + v_6 = self.limit - self.cursor + try: + # call vowel_suffix, line 187 + if not self.r_vowel_suffix(): + raise lab6() + except lab6: pass + self.cursor = self.limit - v_6 + self.cursor = self.limit_backward + # do, line 189 + v_7 = self.cursor + try: + # call postlude, line 189 + if not self.r_postlude(): + raise lab7() + except lab7: pass + self.cursor = v_7 + return True + + def equals(self, o): + return isinstance(o, ItalianStemmer) + + def hashCode(self): + return hash("ItalianStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass +class lab16(BaseException): pass +class lab17(BaseException): pass +class lab18(BaseException): pass +class lab19(BaseException): pass +class lab20(BaseException): pass +class lab21(BaseException): pass diff --git a/contrib/snowballstemmer/norwegian_stemmer.py b/contrib/snowballstemmer/norwegian_stemmer.py new file mode 100644 index 0000000..fad3b76 --- /dev/null +++ b/contrib/snowballstemmer/norwegian_stemmer.py @@ -0,0 +1,308 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class NorwegianStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"a", -1, 1), + Among(u"e", -1, 1), + Among(u"ede", 1, 1), + Among(u"ande", 1, 1), + Among(u"ende", 1, 1), + Among(u"ane", 1, 1), + Among(u"ene", 1, 1), + Among(u"hetene", 6, 1), + Among(u"erte", 1, 3), + Among(u"en", -1, 1), + Among(u"heten", 9, 1), + Among(u"ar", -1, 1), + Among(u"er", -1, 1), + Among(u"heter", 12, 1), + Among(u"s", -1, 2), + Among(u"as", 14, 1), + Among(u"es", 14, 1), + Among(u"edes", 16, 1), + Among(u"endes", 16, 1), + Among(u"enes", 16, 1), + Among(u"hetenes", 19, 1), + Among(u"ens", 14, 1), + Among(u"hetens", 21, 1), + Among(u"ers", 14, 1), + Among(u"ets", 14, 1), + Among(u"et", -1, 1), + Among(u"het", 25, 1), + Among(u"ert", -1, 3), + Among(u"ast", -1, 1) + ] + + a_1 = [ + Among(u"dt", -1, -1), + Among(u"vt", -1, -1) + ] + + a_2 = [ + Among(u"leg", -1, 1), + Among(u"eleg", 0, 1), + Among(u"ig", -1, 1), + Among(u"eig", 2, 1), + Among(u"lig", 2, 1), + Among(u"elig", 4, 1), + Among(u"els", -1, 1), + Among(u"lov", -1, 1), + Among(u"elov", 7, 1), + Among(u"slov", 7, 1), + Among(u"hetslov", 9, 1) + ] + + g_v = [17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 128] + + g_s_ending = [119, 125, 149, 1] + + I_x = 0 + I_p1 = 0 + + def copy_from(self, other): + self.I_x = other.I_x + self.I_p1 = other.I_p1 + super.copy_from(other) + + + def r_mark_regions(self): + # (, line 26 + self.I_p1 = self.limit; + # test, line 30 + v_1 = self.cursor + # (, line 30 + # hop, line 30 + c = self.cursor + 3 + if 0 > c or c > self.limit: + return False + self.cursor = c + # setmark x, line 30 + self.I_x = self.cursor + self.cursor = v_1 + # goto, line 31 + try: + while True: + v_2 = self.cursor + try: + if not self.in_grouping(NorwegianStemmer.g_v, 97, 248): + raise lab1() + self.cursor = v_2 + raise lab0() + except lab1: pass + self.cursor = v_2 + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab0: pass + # gopast, line 31 + try: + while True: + try: + if not self.out_grouping(NorwegianStemmer.g_v, 97, 248): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab2: pass + # setmark p1, line 31 + self.I_p1 = self.cursor + # try, line 32 + try: + # (, line 32 + if not (self.I_p1 < self.I_x): + raise lab4() + self.I_p1 = self.I_x; + except lab4: pass + return True + + def r_main_suffix(self): + # (, line 37 + # setlimit, line 38 + v_1 = self.limit - self.cursor + # tomark, line 38 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 38 + # [, line 38 + self.ket = self.cursor + # substring, line 38 + among_var = self.find_among_b(NorwegianStemmer.a_0, 29) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 38 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 44 + # delete, line 44 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 46 + # or, line 46 + try: + v_3 = self.limit - self.cursor + try: + if not self.in_grouping_b(NorwegianStemmer.g_s_ending, 98, 122): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_3 + # (, line 46 + # literal, line 46 + if not self.eq_s_b(1, u"k"): + return False + if not self.out_grouping_b(NorwegianStemmer.g_v, 97, 248): + return False + except lab0: pass + # delete, line 46 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 48 + # <-, line 48 + if not self.slice_from(u"er"): + return False + return True + + def r_consonant_pair(self): + # (, line 52 + # test, line 53 + v_1 = self.limit - self.cursor + # (, line 53 + # setlimit, line 54 + v_2 = self.limit - self.cursor + # tomark, line 54 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_3 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_2 + # (, line 54 + # [, line 54 + self.ket = self.cursor + # substring, line 54 + if self.find_among_b(NorwegianStemmer.a_1, 2) == 0: + self.limit_backward = v_3 + return False + # ], line 54 + self.bra = self.cursor + self.limit_backward = v_3 + self.cursor = self.limit - v_1 + # next, line 59 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # ], line 59 + self.bra = self.cursor + # delete, line 59 + if not self.slice_del(): + return False + + return True + + def r_other_suffix(self): + # (, line 62 + # setlimit, line 63 + v_1 = self.limit - self.cursor + # tomark, line 63 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 63 + # [, line 63 + self.ket = self.cursor + # substring, line 63 + among_var = self.find_among_b(NorwegianStemmer.a_2, 11) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 63 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 67 + # delete, line 67 + if not self.slice_del(): + return False + + return True + + def _stem(self): + # (, line 72 + # do, line 74 + v_1 = self.cursor + try: + # call mark_regions, line 74 + if not self.r_mark_regions(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # backwards, line 75 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 75 + # do, line 76 + v_2 = self.limit - self.cursor + try: + # call main_suffix, line 76 + if not self.r_main_suffix(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 77 + v_3 = self.limit - self.cursor + try: + # call consonant_pair, line 77 + if not self.r_consonant_pair(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 78 + v_4 = self.limit - self.cursor + try: + # call other_suffix, line 78 + if not self.r_other_suffix(): + raise lab3() + except lab3: pass + self.cursor = self.limit - v_4 + self.cursor = self.limit_backward + return True + + def equals(self, o): + return isinstance(o, NorwegianStemmer) + + def hashCode(self): + return hash("NorwegianStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass diff --git a/contrib/snowballstemmer/porter_stemmer.py b/contrib/snowballstemmer/porter_stemmer.py new file mode 100644 index 0000000..4045629 --- /dev/null +++ b/contrib/snowballstemmer/porter_stemmer.py @@ -0,0 +1,789 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class PorterStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"s", -1, 3), + Among(u"ies", 0, 2), + Among(u"sses", 0, 1), + Among(u"ss", 0, -1) + ] + + a_1 = [ + Among(u"", -1, 3), + Among(u"bb", 0, 2), + Among(u"dd", 0, 2), + Among(u"ff", 0, 2), + Among(u"gg", 0, 2), + Among(u"bl", 0, 1), + Among(u"mm", 0, 2), + Among(u"nn", 0, 2), + Among(u"pp", 0, 2), + Among(u"rr", 0, 2), + Among(u"at", 0, 1), + Among(u"tt", 0, 2), + Among(u"iz", 0, 1) + ] + + a_2 = [ + Among(u"ed", -1, 2), + Among(u"eed", 0, 1), + Among(u"ing", -1, 2) + ] + + a_3 = [ + Among(u"anci", -1, 3), + Among(u"enci", -1, 2), + Among(u"abli", -1, 4), + Among(u"eli", -1, 6), + Among(u"alli", -1, 9), + Among(u"ousli", -1, 12), + Among(u"entli", -1, 5), + Among(u"aliti", -1, 10), + Among(u"biliti", -1, 14), + Among(u"iviti", -1, 13), + Among(u"tional", -1, 1), + Among(u"ational", 10, 8), + Among(u"alism", -1, 10), + Among(u"ation", -1, 8), + Among(u"ization", 13, 7), + Among(u"izer", -1, 7), + Among(u"ator", -1, 8), + Among(u"iveness", -1, 13), + Among(u"fulness", -1, 11), + Among(u"ousness", -1, 12) + ] + + a_4 = [ + Among(u"icate", -1, 2), + Among(u"ative", -1, 3), + Among(u"alize", -1, 1), + Among(u"iciti", -1, 2), + Among(u"ical", -1, 2), + Among(u"ful", -1, 3), + Among(u"ness", -1, 3) + ] + + a_5 = [ + Among(u"ic", -1, 1), + Among(u"ance", -1, 1), + Among(u"ence", -1, 1), + Among(u"able", -1, 1), + Among(u"ible", -1, 1), + Among(u"ate", -1, 1), + Among(u"ive", -1, 1), + Among(u"ize", -1, 1), + Among(u"iti", -1, 1), + Among(u"al", -1, 1), + Among(u"ism", -1, 1), + Among(u"ion", -1, 2), + Among(u"er", -1, 1), + Among(u"ous", -1, 1), + Among(u"ant", -1, 1), + Among(u"ent", -1, 1), + Among(u"ment", 15, 1), + Among(u"ement", 16, 1), + Among(u"ou", -1, 1) + ] + + g_v = [17, 65, 16, 1] + + g_v_WXY = [1, 17, 65, 208, 1] + + B_Y_found = False + I_p2 = 0 + I_p1 = 0 + + def copy_from(self, other): + self.B_Y_found = other.B_Y_found + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + super.copy_from(other) + + + def r_shortv(self): + # (, line 19 + if not self.out_grouping_b(PorterStemmer.g_v_WXY, 89, 121): + return False + if not self.in_grouping_b(PorterStemmer.g_v, 97, 121): + return False + if not self.out_grouping_b(PorterStemmer.g_v, 97, 121): + return False + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_Step_1a(self): + # (, line 24 + # [, line 25 + self.ket = self.cursor + # substring, line 25 + among_var = self.find_among_b(PorterStemmer.a_0, 4) + if among_var == 0: + return False + # ], line 25 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 26 + # <-, line 26 + if not self.slice_from(u"ss"): + return False + elif among_var == 2: + # (, line 27 + # <-, line 27 + if not self.slice_from(u"i"): + return False + elif among_var == 3: + # (, line 29 + # delete, line 29 + if not self.slice_del(): + return False + + return True + + def r_Step_1b(self): + # (, line 33 + # [, line 34 + self.ket = self.cursor + # substring, line 34 + among_var = self.find_among_b(PorterStemmer.a_2, 3) + if among_var == 0: + return False + # ], line 34 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 35 + # call R1, line 35 + if not self.r_R1(): + return False + # <-, line 35 + if not self.slice_from(u"ee"): + return False + elif among_var == 2: + # (, line 37 + # test, line 38 + v_1 = self.limit - self.cursor + # gopast, line 38 + try: + while True: + try: + if not self.in_grouping_b(PorterStemmer.g_v, 97, 121): + raise lab1() + raise lab0() + except lab1: pass + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab0: pass + self.cursor = self.limit - v_1 + # delete, line 38 + if not self.slice_del(): + return False + + # test, line 39 + v_3 = self.limit - self.cursor + # substring, line 39 + among_var = self.find_among_b(PorterStemmer.a_1, 13) + if among_var == 0: + return False + self.cursor = self.limit - v_3 + if among_var == 0: + return False + elif among_var == 1: + # (, line 41 + # <+, line 41 + c = self.cursor + self.insert(self.cursor, self.cursor, u"e") + self.cursor = c + elif among_var == 2: + # (, line 44 + # [, line 44 + self.ket = self.cursor + # next, line 44 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # ], line 44 + self.bra = self.cursor + # delete, line 44 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 45 + # atmark, line 45 + if self.cursor != self.I_p1: + return False + # test, line 45 + v_4 = self.limit - self.cursor + # call shortv, line 45 + if not self.r_shortv(): + return False + self.cursor = self.limit - v_4 + # <+, line 45 + c = self.cursor + self.insert(self.cursor, self.cursor, u"e") + self.cursor = c + return True + + def r_Step_1c(self): + # (, line 51 + # [, line 52 + self.ket = self.cursor + # or, line 52 + try: + v_1 = self.limit - self.cursor + try: + # literal, line 52 + if not self.eq_s_b(1, u"y"): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # literal, line 52 + if not self.eq_s_b(1, u"Y"): + return False + except lab0: pass + # ], line 52 + self.bra = self.cursor + # gopast, line 53 + try: + while True: + try: + if not self.in_grouping_b(PorterStemmer.g_v, 97, 121): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab2: pass + # <-, line 54 + if not self.slice_from(u"i"): + return False + return True + + def r_Step_2(self): + # (, line 57 + # [, line 58 + self.ket = self.cursor + # substring, line 58 + among_var = self.find_among_b(PorterStemmer.a_3, 20) + if among_var == 0: + return False + # ], line 58 + self.bra = self.cursor + # call R1, line 58 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 59 + # <-, line 59 + if not self.slice_from(u"tion"): + return False + elif among_var == 2: + # (, line 60 + # <-, line 60 + if not self.slice_from(u"ence"): + return False + elif among_var == 3: + # (, line 61 + # <-, line 61 + if not self.slice_from(u"ance"): + return False + elif among_var == 4: + # (, line 62 + # <-, line 62 + if not self.slice_from(u"able"): + return False + elif among_var == 5: + # (, line 63 + # <-, line 63 + if not self.slice_from(u"ent"): + return False + elif among_var == 6: + # (, line 64 + # <-, line 64 + if not self.slice_from(u"e"): + return False + elif among_var == 7: + # (, line 66 + # <-, line 66 + if not self.slice_from(u"ize"): + return False + elif among_var == 8: + # (, line 68 + # <-, line 68 + if not self.slice_from(u"ate"): + return False + elif among_var == 9: + # (, line 69 + # <-, line 69 + if not self.slice_from(u"al"): + return False + elif among_var == 10: + # (, line 71 + # <-, line 71 + if not self.slice_from(u"al"): + return False + elif among_var == 11: + # (, line 72 + # <-, line 72 + if not self.slice_from(u"ful"): + return False + elif among_var == 12: + # (, line 74 + # <-, line 74 + if not self.slice_from(u"ous"): + return False + elif among_var == 13: + # (, line 76 + # <-, line 76 + if not self.slice_from(u"ive"): + return False + elif among_var == 14: + # (, line 77 + # <-, line 77 + if not self.slice_from(u"ble"): + return False + return True + + def r_Step_3(self): + # (, line 81 + # [, line 82 + self.ket = self.cursor + # substring, line 82 + among_var = self.find_among_b(PorterStemmer.a_4, 7) + if among_var == 0: + return False + # ], line 82 + self.bra = self.cursor + # call R1, line 82 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 83 + # <-, line 83 + if not self.slice_from(u"al"): + return False + elif among_var == 2: + # (, line 85 + # <-, line 85 + if not self.slice_from(u"ic"): + return False + elif among_var == 3: + # (, line 87 + # delete, line 87 + if not self.slice_del(): + return False + + return True + + def r_Step_4(self): + # (, line 91 + # [, line 92 + self.ket = self.cursor + # substring, line 92 + among_var = self.find_among_b(PorterStemmer.a_5, 19) + if among_var == 0: + return False + # ], line 92 + self.bra = self.cursor + # call R2, line 92 + if not self.r_R2(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 95 + # delete, line 95 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 96 + # or, line 96 + try: + v_1 = self.limit - self.cursor + try: + # literal, line 96 + if not self.eq_s_b(1, u"s"): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # literal, line 96 + if not self.eq_s_b(1, u"t"): + return False + except lab0: pass + # delete, line 96 + if not self.slice_del(): + return False + + return True + + def r_Step_5a(self): + # (, line 100 + # [, line 101 + self.ket = self.cursor + # literal, line 101 + if not self.eq_s_b(1, u"e"): + return False + # ], line 101 + self.bra = self.cursor + # or, line 102 + try: + v_1 = self.limit - self.cursor + try: + # call R2, line 102 + if not self.r_R2(): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 102 + # call R1, line 102 + if not self.r_R1(): + return False + # not, line 102 + v_2 = self.limit - self.cursor + try: + # call shortv, line 102 + if not self.r_shortv(): + raise lab2() + return False + except lab2: pass + self.cursor = self.limit - v_2 + except lab0: pass + # delete, line 103 + if not self.slice_del(): + return False + + return True + + def r_Step_5b(self): + # (, line 106 + # [, line 107 + self.ket = self.cursor + # literal, line 107 + if not self.eq_s_b(1, u"l"): + return False + # ], line 107 + self.bra = self.cursor + # call R2, line 108 + if not self.r_R2(): + return False + # literal, line 108 + if not self.eq_s_b(1, u"l"): + return False + # delete, line 109 + if not self.slice_del(): + return False + + return True + + def _stem(self): + # (, line 113 + # unset Y_found, line 115 + self.B_Y_found = False + # do, line 116 + v_1 = self.cursor + try: + # (, line 116 + # [, line 116 + self.bra = self.cursor + # literal, line 116 + if not self.eq_s(1, u"y"): + raise lab0() + # ], line 116 + self.ket = self.cursor + # <-, line 116 + if not self.slice_from(u"Y"): + return False + # set Y_found, line 116 + self.B_Y_found = True + except lab0: pass + self.cursor = v_1 + # do, line 117 + v_2 = self.cursor + try: + # repeat, line 117 + try: + while True: + try: + v_3 = self.cursor + try: + # (, line 117 + # goto, line 117 + try: + while True: + v_4 = self.cursor + try: + # (, line 117 + if not self.in_grouping(PorterStemmer.g_v, 97, 121): + raise lab6() + # [, line 117 + self.bra = self.cursor + # literal, line 117 + if not self.eq_s(1, u"y"): + raise lab6() + # ], line 117 + self.ket = self.cursor + self.cursor = v_4 + raise lab5() + except lab6: pass + self.cursor = v_4 + if self.cursor >= self.limit: + raise lab4() + self.cursor += 1 + except lab5: pass + # <-, line 117 + if not self.slice_from(u"Y"): + return False + # set Y_found, line 117 + self.B_Y_found = True + raise lab3() + except lab4: pass + self.cursor = v_3 + raise lab2() + except lab3: pass + except lab2: pass + except lab1: pass + self.cursor = v_2 + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # do, line 121 + v_5 = self.cursor + try: + # (, line 121 + # gopast, line 122 + try: + while True: + try: + if not self.in_grouping(PorterStemmer.g_v, 97, 121): + raise lab9() + raise lab8() + except lab9: pass + if self.cursor >= self.limit: + raise lab7() + self.cursor += 1 + except lab8: pass + # gopast, line 122 + try: + while True: + try: + if not self.out_grouping(PorterStemmer.g_v, 97, 121): + raise lab11() + raise lab10() + except lab11: pass + if self.cursor >= self.limit: + raise lab7() + self.cursor += 1 + except lab10: pass + # setmark p1, line 122 + self.I_p1 = self.cursor + # gopast, line 123 + try: + while True: + try: + if not self.in_grouping(PorterStemmer.g_v, 97, 121): + raise lab13() + raise lab12() + except lab13: pass + if self.cursor >= self.limit: + raise lab7() + self.cursor += 1 + except lab12: pass + # gopast, line 123 + try: + while True: + try: + if not self.out_grouping(PorterStemmer.g_v, 97, 121): + raise lab15() + raise lab14() + except lab15: pass + if self.cursor >= self.limit: + raise lab7() + self.cursor += 1 + except lab14: pass + # setmark p2, line 123 + self.I_p2 = self.cursor + except lab7: pass + self.cursor = v_5 + # backwards, line 126 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 126 + # do, line 127 + v_10 = self.limit - self.cursor + try: + # call Step_1a, line 127 + if not self.r_Step_1a(): + raise lab16() + except lab16: pass + self.cursor = self.limit - v_10 + # do, line 128 + v_11 = self.limit - self.cursor + try: + # call Step_1b, line 128 + if not self.r_Step_1b(): + raise lab17() + except lab17: pass + self.cursor = self.limit - v_11 + # do, line 129 + v_12 = self.limit - self.cursor + try: + # call Step_1c, line 129 + if not self.r_Step_1c(): + raise lab18() + except lab18: pass + self.cursor = self.limit - v_12 + # do, line 130 + v_13 = self.limit - self.cursor + try: + # call Step_2, line 130 + if not self.r_Step_2(): + raise lab19() + except lab19: pass + self.cursor = self.limit - v_13 + # do, line 131 + v_14 = self.limit - self.cursor + try: + # call Step_3, line 131 + if not self.r_Step_3(): + raise lab20() + except lab20: pass + self.cursor = self.limit - v_14 + # do, line 132 + v_15 = self.limit - self.cursor + try: + # call Step_4, line 132 + if not self.r_Step_4(): + raise lab21() + except lab21: pass + self.cursor = self.limit - v_15 + # do, line 133 + v_16 = self.limit - self.cursor + try: + # call Step_5a, line 133 + if not self.r_Step_5a(): + raise lab22() + except lab22: pass + self.cursor = self.limit - v_16 + # do, line 134 + v_17 = self.limit - self.cursor + try: + # call Step_5b, line 134 + if not self.r_Step_5b(): + raise lab23() + except lab23: pass + self.cursor = self.limit - v_17 + self.cursor = self.limit_backward + # do, line 137 + v_18 = self.cursor + try: + # (, line 137 + # Boolean test Y_found, line 137 + if not self.B_Y_found: + raise lab24() + # repeat, line 137 + try: + while True: + try: + v_19 = self.cursor + try: + # (, line 137 + # goto, line 137 + try: + while True: + v_20 = self.cursor + try: + # (, line 137 + # [, line 137 + self.bra = self.cursor + # literal, line 137 + if not self.eq_s(1, u"Y"): + raise lab29() + # ], line 137 + self.ket = self.cursor + self.cursor = v_20 + raise lab28() + except lab29: pass + self.cursor = v_20 + if self.cursor >= self.limit: + raise lab27() + self.cursor += 1 + except lab28: pass + # <-, line 137 + if not self.slice_from(u"y"): + return False + raise lab26() + except lab27: pass + self.cursor = v_19 + raise lab25() + except lab26: pass + except lab25: pass + except lab24: pass + self.cursor = v_18 + return True + + def equals(self, o): + return isinstance(o, PorterStemmer) + + def hashCode(self): + return hash("PorterStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass +class lab16(BaseException): pass +class lab17(BaseException): pass +class lab18(BaseException): pass +class lab19(BaseException): pass +class lab20(BaseException): pass +class lab21(BaseException): pass +class lab22(BaseException): pass +class lab23(BaseException): pass +class lab24(BaseException): pass +class lab25(BaseException): pass +class lab26(BaseException): pass +class lab27(BaseException): pass +class lab28(BaseException): pass +class lab29(BaseException): pass diff --git a/contrib/snowballstemmer/portuguese_stemmer.py b/contrib/snowballstemmer/portuguese_stemmer.py new file mode 100644 index 0000000..08d9697 --- /dev/null +++ b/contrib/snowballstemmer/portuguese_stemmer.py @@ -0,0 +1,965 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class PortugueseStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"", -1, 3), + Among(u"\u00E3", 0, 1), + Among(u"\u00F5", 0, 2) + ] + + a_1 = [ + Among(u"", -1, 3), + Among(u"a~", 0, 1), + Among(u"o~", 0, 2) + ] + + a_2 = [ + Among(u"ic", -1, -1), + Among(u"ad", -1, -1), + Among(u"os", -1, -1), + Among(u"iv", -1, 1) + ] + + a_3 = [ + Among(u"ante", -1, 1), + Among(u"avel", -1, 1), + Among(u"\u00EDvel", -1, 1) + ] + + a_4 = [ + Among(u"ic", -1, 1), + Among(u"abil", -1, 1), + Among(u"iv", -1, 1) + ] + + a_5 = [ + Among(u"ica", -1, 1), + Among(u"\u00E2ncia", -1, 1), + Among(u"\u00EAncia", -1, 4), + Among(u"ira", -1, 9), + Among(u"adora", -1, 1), + Among(u"osa", -1, 1), + Among(u"ista", -1, 1), + Among(u"iva", -1, 8), + Among(u"eza", -1, 1), + Among(u"log\u00EDa", -1, 2), + Among(u"idade", -1, 7), + Among(u"ante", -1, 1), + Among(u"mente", -1, 6), + Among(u"amente", 12, 5), + Among(u"\u00E1vel", -1, 1), + Among(u"\u00EDvel", -1, 1), + Among(u"uci\u00F3n", -1, 3), + Among(u"ico", -1, 1), + Among(u"ismo", -1, 1), + Among(u"oso", -1, 1), + Among(u"amento", -1, 1), + Among(u"imento", -1, 1), + Among(u"ivo", -1, 8), + Among(u"a\u00E7a~o", -1, 1), + Among(u"ador", -1, 1), + Among(u"icas", -1, 1), + Among(u"\u00EAncias", -1, 4), + Among(u"iras", -1, 9), + Among(u"adoras", -1, 1), + Among(u"osas", -1, 1), + Among(u"istas", -1, 1), + Among(u"ivas", -1, 8), + Among(u"ezas", -1, 1), + Among(u"log\u00EDas", -1, 2), + Among(u"idades", -1, 7), + Among(u"uciones", -1, 3), + Among(u"adores", -1, 1), + Among(u"antes", -1, 1), + Among(u"a\u00E7o~es", -1, 1), + Among(u"icos", -1, 1), + Among(u"ismos", -1, 1), + Among(u"osos", -1, 1), + Among(u"amentos", -1, 1), + Among(u"imentos", -1, 1), + Among(u"ivos", -1, 8) + ] + + a_6 = [ + Among(u"ada", -1, 1), + Among(u"ida", -1, 1), + Among(u"ia", -1, 1), + Among(u"aria", 2, 1), + Among(u"eria", 2, 1), + Among(u"iria", 2, 1), + Among(u"ara", -1, 1), + Among(u"era", -1, 1), + Among(u"ira", -1, 1), + Among(u"ava", -1, 1), + Among(u"asse", -1, 1), + Among(u"esse", -1, 1), + Among(u"isse", -1, 1), + Among(u"aste", -1, 1), + Among(u"este", -1, 1), + Among(u"iste", -1, 1), + Among(u"ei", -1, 1), + Among(u"arei", 16, 1), + Among(u"erei", 16, 1), + Among(u"irei", 16, 1), + Among(u"am", -1, 1), + Among(u"iam", 20, 1), + Among(u"ariam", 21, 1), + Among(u"eriam", 21, 1), + Among(u"iriam", 21, 1), + Among(u"aram", 20, 1), + Among(u"eram", 20, 1), + Among(u"iram", 20, 1), + Among(u"avam", 20, 1), + Among(u"em", -1, 1), + Among(u"arem", 29, 1), + Among(u"erem", 29, 1), + Among(u"irem", 29, 1), + Among(u"assem", 29, 1), + Among(u"essem", 29, 1), + Among(u"issem", 29, 1), + Among(u"ado", -1, 1), + Among(u"ido", -1, 1), + Among(u"ando", -1, 1), + Among(u"endo", -1, 1), + Among(u"indo", -1, 1), + Among(u"ara~o", -1, 1), + Among(u"era~o", -1, 1), + Among(u"ira~o", -1, 1), + Among(u"ar", -1, 1), + Among(u"er", -1, 1), + Among(u"ir", -1, 1), + Among(u"as", -1, 1), + Among(u"adas", 47, 1), + Among(u"idas", 47, 1), + Among(u"ias", 47, 1), + Among(u"arias", 50, 1), + Among(u"erias", 50, 1), + Among(u"irias", 50, 1), + Among(u"aras", 47, 1), + Among(u"eras", 47, 1), + Among(u"iras", 47, 1), + Among(u"avas", 47, 1), + Among(u"es", -1, 1), + Among(u"ardes", 58, 1), + Among(u"erdes", 58, 1), + Among(u"irdes", 58, 1), + Among(u"ares", 58, 1), + Among(u"eres", 58, 1), + Among(u"ires", 58, 1), + Among(u"asses", 58, 1), + Among(u"esses", 58, 1), + Among(u"isses", 58, 1), + Among(u"astes", 58, 1), + Among(u"estes", 58, 1), + Among(u"istes", 58, 1), + Among(u"is", -1, 1), + Among(u"ais", 71, 1), + Among(u"eis", 71, 1), + Among(u"areis", 73, 1), + Among(u"ereis", 73, 1), + Among(u"ireis", 73, 1), + Among(u"\u00E1reis", 73, 1), + Among(u"\u00E9reis", 73, 1), + Among(u"\u00EDreis", 73, 1), + Among(u"\u00E1sseis", 73, 1), + Among(u"\u00E9sseis", 73, 1), + Among(u"\u00EDsseis", 73, 1), + Among(u"\u00E1veis", 73, 1), + Among(u"\u00EDeis", 73, 1), + Among(u"ar\u00EDeis", 84, 1), + Among(u"er\u00EDeis", 84, 1), + Among(u"ir\u00EDeis", 84, 1), + Among(u"ados", -1, 1), + Among(u"idos", -1, 1), + Among(u"amos", -1, 1), + Among(u"\u00E1ramos", 90, 1), + Among(u"\u00E9ramos", 90, 1), + Among(u"\u00EDramos", 90, 1), + Among(u"\u00E1vamos", 90, 1), + Among(u"\u00EDamos", 90, 1), + Among(u"ar\u00EDamos", 95, 1), + Among(u"er\u00EDamos", 95, 1), + Among(u"ir\u00EDamos", 95, 1), + Among(u"emos", -1, 1), + Among(u"aremos", 99, 1), + Among(u"eremos", 99, 1), + Among(u"iremos", 99, 1), + Among(u"\u00E1ssemos", 99, 1), + Among(u"\u00EAssemos", 99, 1), + Among(u"\u00EDssemos", 99, 1), + Among(u"imos", -1, 1), + Among(u"armos", -1, 1), + Among(u"ermos", -1, 1), + Among(u"irmos", -1, 1), + Among(u"\u00E1mos", -1, 1), + Among(u"ar\u00E1s", -1, 1), + Among(u"er\u00E1s", -1, 1), + Among(u"ir\u00E1s", -1, 1), + Among(u"eu", -1, 1), + Among(u"iu", -1, 1), + Among(u"ou", -1, 1), + Among(u"ar\u00E1", -1, 1), + Among(u"er\u00E1", -1, 1), + Among(u"ir\u00E1", -1, 1) + ] + + a_7 = [ + Among(u"a", -1, 1), + Among(u"i", -1, 1), + Among(u"o", -1, 1), + Among(u"os", -1, 1), + Among(u"\u00E1", -1, 1), + Among(u"\u00ED", -1, 1), + Among(u"\u00F3", -1, 1) + ] + + a_8 = [ + Among(u"e", -1, 1), + Among(u"\u00E7", -1, 2), + Among(u"\u00E9", -1, 1), + Among(u"\u00EA", -1, 1) + ] + + g_v = [17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 19, 12, 2] + + I_p2 = 0 + I_p1 = 0 + I_pV = 0 + + def copy_from(self, other): + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + self.I_pV = other.I_pV + super.copy_from(other) + + + def r_prelude(self): + # repeat, line 36 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 36 + # [, line 37 + self.bra = self.cursor + # substring, line 37 + among_var = self.find_among(PortugueseStemmer.a_0, 3) + if among_var == 0: + raise lab2() + # ], line 37 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 38 + # <-, line 38 + if not self.slice_from(u"a~"): + return False + elif among_var == 2: + # (, line 39 + # <-, line 39 + if not self.slice_from(u"o~"): + return False + elif among_var == 3: + # (, line 40 + # next, line 40 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_mark_regions(self): + # (, line 44 + self.I_pV = self.limit; + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # do, line 50 + v_1 = self.cursor + try: + # (, line 50 + # or, line 52 + try: + v_2 = self.cursor + try: + # (, line 51 + if not self.in_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab2() + # or, line 51 + try: + v_3 = self.cursor + try: + # (, line 51 + if not self.out_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab4() + # gopast, line 51 + try: + while True: + try: + if not self.in_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + raise lab4() + self.cursor += 1 + except lab5: pass + raise lab3() + except lab4: pass + self.cursor = v_3 + # (, line 51 + if not self.in_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab2() + # gopast, line 51 + try: + while True: + try: + if not self.out_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab7: pass + except lab3: pass + raise lab1() + except lab2: pass + self.cursor = v_2 + # (, line 53 + if not self.out_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab0() + # or, line 53 + try: + v_6 = self.cursor + try: + # (, line 53 + if not self.out_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab10() + # gopast, line 53 + try: + while True: + try: + if not self.in_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab12() + raise lab11() + except lab12: pass + if self.cursor >= self.limit: + raise lab10() + self.cursor += 1 + except lab11: pass + raise lab9() + except lab10: pass + self.cursor = v_6 + # (, line 53 + if not self.in_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab0() + # next, line 53 + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab9: pass + except lab1: pass + # setmark pV, line 54 + self.I_pV = self.cursor + except lab0: pass + self.cursor = v_1 + # do, line 56 + v_8 = self.cursor + try: + # (, line 56 + # gopast, line 57 + try: + while True: + try: + if not self.in_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab15() + raise lab14() + except lab15: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab14: pass + # gopast, line 57 + try: + while True: + try: + if not self.out_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab17() + raise lab16() + except lab17: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab16: pass + # setmark p1, line 57 + self.I_p1 = self.cursor + # gopast, line 58 + try: + while True: + try: + if not self.in_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab19() + raise lab18() + except lab19: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab18: pass + # gopast, line 58 + try: + while True: + try: + if not self.out_grouping(PortugueseStemmer.g_v, 97, 250): + raise lab21() + raise lab20() + except lab21: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab20: pass + # setmark p2, line 58 + self.I_p2 = self.cursor + except lab13: pass + self.cursor = v_8 + return True + + def r_postlude(self): + # repeat, line 62 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 62 + # [, line 63 + self.bra = self.cursor + # substring, line 63 + among_var = self.find_among(PortugueseStemmer.a_1, 3) + if among_var == 0: + raise lab2() + # ], line 63 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 64 + # <-, line 64 + if not self.slice_from(u"\u00E3"): + return False + elif among_var == 2: + # (, line 65 + # <-, line 65 + if not self.slice_from(u"\u00F5"): + return False + elif among_var == 3: + # (, line 66 + # next, line 66 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_RV(self): + if not self.I_pV <= self.cursor: + return False + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_standard_suffix(self): + # (, line 76 + # [, line 77 + self.ket = self.cursor + # substring, line 77 + among_var = self.find_among_b(PortugueseStemmer.a_5, 45) + if among_var == 0: + return False + # ], line 77 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 92 + # call R2, line 93 + if not self.r_R2(): + return False + # delete, line 93 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 97 + # call R2, line 98 + if not self.r_R2(): + return False + # <-, line 98 + if not self.slice_from(u"log"): + return False + elif among_var == 3: + # (, line 101 + # call R2, line 102 + if not self.r_R2(): + return False + # <-, line 102 + if not self.slice_from(u"u"): + return False + elif among_var == 4: + # (, line 105 + # call R2, line 106 + if not self.r_R2(): + return False + # <-, line 106 + if not self.slice_from(u"ente"): + return False + elif among_var == 5: + # (, line 109 + # call R1, line 110 + if not self.r_R1(): + return False + # delete, line 110 + if not self.slice_del(): + return False + + # try, line 111 + v_1 = self.limit - self.cursor + try: + # (, line 111 + # [, line 112 + self.ket = self.cursor + # substring, line 112 + among_var = self.find_among_b(PortugueseStemmer.a_2, 4) + if among_var == 0: + self.cursor = self.limit - v_1 + raise lab0() + # ], line 112 + self.bra = self.cursor + # call R2, line 112 + if not self.r_R2(): + self.cursor = self.limit - v_1 + raise lab0() + # delete, line 112 + if not self.slice_del(): + return False + + if among_var == 0: + self.cursor = self.limit - v_1 + raise lab0() + elif among_var == 1: + # (, line 113 + # [, line 113 + self.ket = self.cursor + # literal, line 113 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 113 + self.bra = self.cursor + # call R2, line 113 + if not self.r_R2(): + self.cursor = self.limit - v_1 + raise lab0() + # delete, line 113 + if not self.slice_del(): + return False + + except lab0: pass + elif among_var == 6: + # (, line 121 + # call R2, line 122 + if not self.r_R2(): + return False + # delete, line 122 + if not self.slice_del(): + return False + + # try, line 123 + v_2 = self.limit - self.cursor + try: + # (, line 123 + # [, line 124 + self.ket = self.cursor + # substring, line 124 + among_var = self.find_among_b(PortugueseStemmer.a_3, 3) + if among_var == 0: + self.cursor = self.limit - v_2 + raise lab1() + # ], line 124 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_2 + raise lab1() + elif among_var == 1: + # (, line 127 + # call R2, line 127 + if not self.r_R2(): + self.cursor = self.limit - v_2 + raise lab1() + # delete, line 127 + if not self.slice_del(): + return False + + except lab1: pass + elif among_var == 7: + # (, line 133 + # call R2, line 134 + if not self.r_R2(): + return False + # delete, line 134 + if not self.slice_del(): + return False + + # try, line 135 + v_3 = self.limit - self.cursor + try: + # (, line 135 + # [, line 136 + self.ket = self.cursor + # substring, line 136 + among_var = self.find_among_b(PortugueseStemmer.a_4, 3) + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab2() + # ], line 136 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab2() + elif among_var == 1: + # (, line 139 + # call R2, line 139 + if not self.r_R2(): + self.cursor = self.limit - v_3 + raise lab2() + # delete, line 139 + if not self.slice_del(): + return False + + except lab2: pass + elif among_var == 8: + # (, line 145 + # call R2, line 146 + if not self.r_R2(): + return False + # delete, line 146 + if not self.slice_del(): + return False + + # try, line 147 + v_4 = self.limit - self.cursor + try: + # (, line 147 + # [, line 148 + self.ket = self.cursor + # literal, line 148 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_4 + raise lab3() + # ], line 148 + self.bra = self.cursor + # call R2, line 148 + if not self.r_R2(): + self.cursor = self.limit - v_4 + raise lab3() + # delete, line 148 + if not self.slice_del(): + return False + + except lab3: pass + elif among_var == 9: + # (, line 152 + # call RV, line 153 + if not self.r_RV(): + return False + # literal, line 153 + if not self.eq_s_b(1, u"e"): + return False + # <-, line 154 + if not self.slice_from(u"ir"): + return False + return True + + def r_verb_suffix(self): + # setlimit, line 159 + v_1 = self.limit - self.cursor + # tomark, line 159 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 159 + # [, line 160 + self.ket = self.cursor + # substring, line 160 + among_var = self.find_among_b(PortugueseStemmer.a_6, 120) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 160 + self.bra = self.cursor + if among_var == 0: + self.limit_backward = v_2 + return False + elif among_var == 1: + # (, line 179 + # delete, line 179 + if not self.slice_del(): + return False + + self.limit_backward = v_2 + return True + + def r_residual_suffix(self): + # (, line 183 + # [, line 184 + self.ket = self.cursor + # substring, line 184 + among_var = self.find_among_b(PortugueseStemmer.a_7, 7) + if among_var == 0: + return False + # ], line 184 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 187 + # call RV, line 187 + if not self.r_RV(): + return False + # delete, line 187 + if not self.slice_del(): + return False + + return True + + def r_residual_form(self): + # (, line 191 + # [, line 192 + self.ket = self.cursor + # substring, line 192 + among_var = self.find_among_b(PortugueseStemmer.a_8, 4) + if among_var == 0: + return False + # ], line 192 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 194 + # call RV, line 194 + if not self.r_RV(): + return False + # delete, line 194 + if not self.slice_del(): + return False + + # [, line 194 + self.ket = self.cursor + # or, line 194 + try: + v_1 = self.limit - self.cursor + try: + # (, line 194 + # literal, line 194 + if not self.eq_s_b(1, u"u"): + raise lab1() + # ], line 194 + self.bra = self.cursor + # test, line 194 + v_2 = self.limit - self.cursor + # literal, line 194 + if not self.eq_s_b(1, u"g"): + raise lab1() + self.cursor = self.limit - v_2 + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 195 + # literal, line 195 + if not self.eq_s_b(1, u"i"): + return False + # ], line 195 + self.bra = self.cursor + # test, line 195 + v_3 = self.limit - self.cursor + # literal, line 195 + if not self.eq_s_b(1, u"c"): + return False + self.cursor = self.limit - v_3 + except lab0: pass + # call RV, line 195 + if not self.r_RV(): + return False + # delete, line 195 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 196 + # <-, line 196 + if not self.slice_from(u"c"): + return False + return True + + def _stem(self): + # (, line 201 + # do, line 202 + v_1 = self.cursor + try: + # call prelude, line 202 + if not self.r_prelude(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # do, line 203 + v_2 = self.cursor + try: + # call mark_regions, line 203 + if not self.r_mark_regions(): + raise lab1() + except lab1: pass + self.cursor = v_2 + # backwards, line 204 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 204 + # do, line 205 + v_3 = self.limit - self.cursor + try: + # (, line 205 + # or, line 209 + try: + v_4 = self.limit - self.cursor + try: + # (, line 206 + # and, line 207 + v_5 = self.limit - self.cursor + # (, line 206 + # or, line 206 + try: + v_6 = self.limit - self.cursor + try: + # call standard_suffix, line 206 + if not self.r_standard_suffix(): + raise lab6() + raise lab5() + except lab6: pass + self.cursor = self.limit - v_6 + # call verb_suffix, line 206 + if not self.r_verb_suffix(): + raise lab4() + except lab5: pass + self.cursor = self.limit - v_5 + # do, line 207 + v_7 = self.limit - self.cursor + try: + # (, line 207 + # [, line 207 + self.ket = self.cursor + # literal, line 207 + if not self.eq_s_b(1, u"i"): + raise lab7() + # ], line 207 + self.bra = self.cursor + # test, line 207 + v_8 = self.limit - self.cursor + # literal, line 207 + if not self.eq_s_b(1, u"c"): + raise lab7() + self.cursor = self.limit - v_8 + # call RV, line 207 + if not self.r_RV(): + raise lab7() + # delete, line 207 + if not self.slice_del(): + return False + + except lab7: pass + self.cursor = self.limit - v_7 + raise lab3() + except lab4: pass + self.cursor = self.limit - v_4 + # call residual_suffix, line 209 + if not self.r_residual_suffix(): + raise lab2() + except lab3: pass + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 211 + v_9 = self.limit - self.cursor + try: + # call residual_form, line 211 + if not self.r_residual_form(): + raise lab8() + except lab8: pass + self.cursor = self.limit - v_9 + self.cursor = self.limit_backward + # do, line 213 + v_10 = self.cursor + try: + # call postlude, line 213 + if not self.r_postlude(): + raise lab9() + except lab9: pass + self.cursor = v_10 + return True + + def equals(self, o): + return isinstance(o, PortugueseStemmer) + + def hashCode(self): + return hash("PortugueseStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass +class lab16(BaseException): pass +class lab17(BaseException): pass +class lab18(BaseException): pass +class lab19(BaseException): pass +class lab20(BaseException): pass +class lab21(BaseException): pass diff --git a/contrib/snowballstemmer/romanian_stemmer.py b/contrib/snowballstemmer/romanian_stemmer.py new file mode 100644 index 0000000..8c95d99 --- /dev/null +++ b/contrib/snowballstemmer/romanian_stemmer.py @@ -0,0 +1,900 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class RomanianStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"", -1, 3), + Among(u"I", 0, 1), + Among(u"U", 0, 2) + ] + + a_1 = [ + Among(u"ea", -1, 3), + Among(u"a\u0163ia", -1, 7), + Among(u"aua", -1, 2), + Among(u"iua", -1, 4), + Among(u"a\u0163ie", -1, 7), + Among(u"ele", -1, 3), + Among(u"ile", -1, 5), + Among(u"iile", 6, 4), + Among(u"iei", -1, 4), + Among(u"atei", -1, 6), + Among(u"ii", -1, 4), + Among(u"ului", -1, 1), + Among(u"ul", -1, 1), + Among(u"elor", -1, 3), + Among(u"ilor", -1, 4), + Among(u"iilor", 14, 4) + ] + + a_2 = [ + Among(u"icala", -1, 4), + Among(u"iciva", -1, 4), + Among(u"ativa", -1, 5), + Among(u"itiva", -1, 6), + Among(u"icale", -1, 4), + Among(u"a\u0163iune", -1, 5), + Among(u"i\u0163iune", -1, 6), + Among(u"atoare", -1, 5), + Among(u"itoare", -1, 6), + Among(u"\u0103toare", -1, 5), + Among(u"icitate", -1, 4), + Among(u"abilitate", -1, 1), + Among(u"ibilitate", -1, 2), + Among(u"ivitate", -1, 3), + Among(u"icive", -1, 4), + Among(u"ative", -1, 5), + Among(u"itive", -1, 6), + Among(u"icali", -1, 4), + Among(u"atori", -1, 5), + Among(u"icatori", 18, 4), + Among(u"itori", -1, 6), + Among(u"\u0103tori", -1, 5), + Among(u"icitati", -1, 4), + Among(u"abilitati", -1, 1), + Among(u"ivitati", -1, 3), + Among(u"icivi", -1, 4), + Among(u"ativi", -1, 5), + Among(u"itivi", -1, 6), + Among(u"icit\u0103i", -1, 4), + Among(u"abilit\u0103i", -1, 1), + Among(u"ivit\u0103i", -1, 3), + Among(u"icit\u0103\u0163i", -1, 4), + Among(u"abilit\u0103\u0163i", -1, 1), + Among(u"ivit\u0103\u0163i", -1, 3), + Among(u"ical", -1, 4), + Among(u"ator", -1, 5), + Among(u"icator", 35, 4), + Among(u"itor", -1, 6), + Among(u"\u0103tor", -1, 5), + Among(u"iciv", -1, 4), + Among(u"ativ", -1, 5), + Among(u"itiv", -1, 6), + Among(u"ical\u0103", -1, 4), + Among(u"iciv\u0103", -1, 4), + Among(u"ativ\u0103", -1, 5), + Among(u"itiv\u0103", -1, 6) + ] + + a_3 = [ + Among(u"ica", -1, 1), + Among(u"abila", -1, 1), + Among(u"ibila", -1, 1), + Among(u"oasa", -1, 1), + Among(u"ata", -1, 1), + Among(u"ita", -1, 1), + Among(u"anta", -1, 1), + Among(u"ista", -1, 3), + Among(u"uta", -1, 1), + Among(u"iva", -1, 1), + Among(u"ic", -1, 1), + Among(u"ice", -1, 1), + Among(u"abile", -1, 1), + Among(u"ibile", -1, 1), + Among(u"isme", -1, 3), + Among(u"iune", -1, 2), + Among(u"oase", -1, 1), + Among(u"ate", -1, 1), + Among(u"itate", 17, 1), + Among(u"ite", -1, 1), + Among(u"ante", -1, 1), + Among(u"iste", -1, 3), + Among(u"ute", -1, 1), + Among(u"ive", -1, 1), + Among(u"ici", -1, 1), + Among(u"abili", -1, 1), + Among(u"ibili", -1, 1), + Among(u"iuni", -1, 2), + Among(u"atori", -1, 1), + Among(u"osi", -1, 1), + Among(u"ati", -1, 1), + Among(u"itati", 30, 1), + Among(u"iti", -1, 1), + Among(u"anti", -1, 1), + Among(u"isti", -1, 3), + Among(u"uti", -1, 1), + Among(u"i\u015Fti", -1, 3), + Among(u"ivi", -1, 1), + Among(u"it\u0103i", -1, 1), + Among(u"o\u015Fi", -1, 1), + Among(u"it\u0103\u0163i", -1, 1), + Among(u"abil", -1, 1), + Among(u"ibil", -1, 1), + Among(u"ism", -1, 3), + Among(u"ator", -1, 1), + Among(u"os", -1, 1), + Among(u"at", -1, 1), + Among(u"it", -1, 1), + Among(u"ant", -1, 1), + Among(u"ist", -1, 3), + Among(u"ut", -1, 1), + Among(u"iv", -1, 1), + Among(u"ic\u0103", -1, 1), + Among(u"abil\u0103", -1, 1), + Among(u"ibil\u0103", -1, 1), + Among(u"oas\u0103", -1, 1), + Among(u"at\u0103", -1, 1), + Among(u"it\u0103", -1, 1), + Among(u"ant\u0103", -1, 1), + Among(u"ist\u0103", -1, 3), + Among(u"ut\u0103", -1, 1), + Among(u"iv\u0103", -1, 1) + ] + + a_4 = [ + Among(u"ea", -1, 1), + Among(u"ia", -1, 1), + Among(u"esc", -1, 1), + Among(u"\u0103sc", -1, 1), + Among(u"ind", -1, 1), + Among(u"\u00E2nd", -1, 1), + Among(u"are", -1, 1), + Among(u"ere", -1, 1), + Among(u"ire", -1, 1), + Among(u"\u00E2re", -1, 1), + Among(u"se", -1, 2), + Among(u"ase", 10, 1), + Among(u"sese", 10, 2), + Among(u"ise", 10, 1), + Among(u"use", 10, 1), + Among(u"\u00E2se", 10, 1), + Among(u"e\u015Fte", -1, 1), + Among(u"\u0103\u015Fte", -1, 1), + Among(u"eze", -1, 1), + Among(u"ai", -1, 1), + Among(u"eai", 19, 1), + Among(u"iai", 19, 1), + Among(u"sei", -1, 2), + Among(u"e\u015Fti", -1, 1), + Among(u"\u0103\u015Fti", -1, 1), + Among(u"ui", -1, 1), + Among(u"ezi", -1, 1), + Among(u"\u00E2i", -1, 1), + Among(u"a\u015Fi", -1, 1), + Among(u"se\u015Fi", -1, 2), + Among(u"ase\u015Fi", 29, 1), + Among(u"sese\u015Fi", 29, 2), + Among(u"ise\u015Fi", 29, 1), + Among(u"use\u015Fi", 29, 1), + Among(u"\u00E2se\u015Fi", 29, 1), + Among(u"i\u015Fi", -1, 1), + Among(u"u\u015Fi", -1, 1), + Among(u"\u00E2\u015Fi", -1, 1), + Among(u"a\u0163i", -1, 2), + Among(u"ea\u0163i", 38, 1), + Among(u"ia\u0163i", 38, 1), + Among(u"e\u0163i", -1, 2), + Among(u"i\u0163i", -1, 2), + Among(u"\u00E2\u0163i", -1, 2), + Among(u"ar\u0103\u0163i", -1, 1), + Among(u"ser\u0103\u0163i", -1, 2), + Among(u"aser\u0103\u0163i", 45, 1), + Among(u"seser\u0103\u0163i", 45, 2), + Among(u"iser\u0103\u0163i", 45, 1), + Among(u"user\u0103\u0163i", 45, 1), + Among(u"\u00E2ser\u0103\u0163i", 45, 1), + Among(u"ir\u0103\u0163i", -1, 1), + Among(u"ur\u0103\u0163i", -1, 1), + Among(u"\u00E2r\u0103\u0163i", -1, 1), + Among(u"am", -1, 1), + Among(u"eam", 54, 1), + Among(u"iam", 54, 1), + Among(u"em", -1, 2), + Among(u"asem", 57, 1), + Among(u"sesem", 57, 2), + Among(u"isem", 57, 1), + Among(u"usem", 57, 1), + Among(u"\u00E2sem", 57, 1), + Among(u"im", -1, 2), + Among(u"\u00E2m", -1, 2), + Among(u"\u0103m", -1, 2), + Among(u"ar\u0103m", 65, 1), + Among(u"ser\u0103m", 65, 2), + Among(u"aser\u0103m", 67, 1), + Among(u"seser\u0103m", 67, 2), + Among(u"iser\u0103m", 67, 1), + Among(u"user\u0103m", 67, 1), + Among(u"\u00E2ser\u0103m", 67, 1), + Among(u"ir\u0103m", 65, 1), + Among(u"ur\u0103m", 65, 1), + Among(u"\u00E2r\u0103m", 65, 1), + Among(u"au", -1, 1), + Among(u"eau", 76, 1), + Among(u"iau", 76, 1), + Among(u"indu", -1, 1), + Among(u"\u00E2ndu", -1, 1), + Among(u"ez", -1, 1), + Among(u"easc\u0103", -1, 1), + Among(u"ar\u0103", -1, 1), + Among(u"ser\u0103", -1, 2), + Among(u"aser\u0103", 84, 1), + Among(u"seser\u0103", 84, 2), + Among(u"iser\u0103", 84, 1), + Among(u"user\u0103", 84, 1), + Among(u"\u00E2ser\u0103", 84, 1), + Among(u"ir\u0103", -1, 1), + Among(u"ur\u0103", -1, 1), + Among(u"\u00E2r\u0103", -1, 1), + Among(u"eaz\u0103", -1, 1) + ] + + a_5 = [ + Among(u"a", -1, 1), + Among(u"e", -1, 1), + Among(u"ie", 1, 1), + Among(u"i", -1, 1), + Among(u"\u0103", -1, 1) + ] + + g_v = [17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4] + + B_standard_suffix_removed = False + I_p2 = 0 + I_p1 = 0 + I_pV = 0 + + def copy_from(self, other): + self.B_standard_suffix_removed = other.B_standard_suffix_removed + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + self.I_pV = other.I_pV + super.copy_from(other) + + + def r_prelude(self): + # (, line 31 + # repeat, line 32 + try: + while True: + try: + v_1 = self.cursor + try: + # goto, line 32 + try: + while True: + v_2 = self.cursor + try: + # (, line 32 + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab4() + # [, line 33 + self.bra = self.cursor + # or, line 33 + try: + v_3 = self.cursor + try: + # (, line 33 + # literal, line 33 + if not self.eq_s(1, u"u"): + raise lab6() + # ], line 33 + self.ket = self.cursor + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab6() + # <-, line 33 + if not self.slice_from(u"U"): + return False + raise lab5() + except lab6: pass + self.cursor = v_3 + # (, line 34 + # literal, line 34 + if not self.eq_s(1, u"i"): + raise lab4() + # ], line 34 + self.ket = self.cursor + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab4() + # <-, line 34 + if not self.slice_from(u"I"): + return False + except lab5: pass + self.cursor = v_2 + raise lab3() + except lab4: pass + self.cursor = v_2 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab3: pass + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_mark_regions(self): + # (, line 38 + self.I_pV = self.limit; + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # do, line 44 + v_1 = self.cursor + try: + # (, line 44 + # or, line 46 + try: + v_2 = self.cursor + try: + # (, line 45 + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab2() + # or, line 45 + try: + v_3 = self.cursor + try: + # (, line 45 + if not self.out_grouping(RomanianStemmer.g_v, 97, 259): + raise lab4() + # gopast, line 45 + try: + while True: + try: + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + raise lab4() + self.cursor += 1 + except lab5: pass + raise lab3() + except lab4: pass + self.cursor = v_3 + # (, line 45 + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab2() + # gopast, line 45 + try: + while True: + try: + if not self.out_grouping(RomanianStemmer.g_v, 97, 259): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab7: pass + except lab3: pass + raise lab1() + except lab2: pass + self.cursor = v_2 + # (, line 47 + if not self.out_grouping(RomanianStemmer.g_v, 97, 259): + raise lab0() + # or, line 47 + try: + v_6 = self.cursor + try: + # (, line 47 + if not self.out_grouping(RomanianStemmer.g_v, 97, 259): + raise lab10() + # gopast, line 47 + try: + while True: + try: + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab12() + raise lab11() + except lab12: pass + if self.cursor >= self.limit: + raise lab10() + self.cursor += 1 + except lab11: pass + raise lab9() + except lab10: pass + self.cursor = v_6 + # (, line 47 + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab0() + # next, line 47 + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab9: pass + except lab1: pass + # setmark pV, line 48 + self.I_pV = self.cursor + except lab0: pass + self.cursor = v_1 + # do, line 50 + v_8 = self.cursor + try: + # (, line 50 + # gopast, line 51 + try: + while True: + try: + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab15() + raise lab14() + except lab15: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab14: pass + # gopast, line 51 + try: + while True: + try: + if not self.out_grouping(RomanianStemmer.g_v, 97, 259): + raise lab17() + raise lab16() + except lab17: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab16: pass + # setmark p1, line 51 + self.I_p1 = self.cursor + # gopast, line 52 + try: + while True: + try: + if not self.in_grouping(RomanianStemmer.g_v, 97, 259): + raise lab19() + raise lab18() + except lab19: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab18: pass + # gopast, line 52 + try: + while True: + try: + if not self.out_grouping(RomanianStemmer.g_v, 97, 259): + raise lab21() + raise lab20() + except lab21: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab20: pass + # setmark p2, line 52 + self.I_p2 = self.cursor + except lab13: pass + self.cursor = v_8 + return True + + def r_postlude(self): + # repeat, line 56 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 56 + # [, line 58 + self.bra = self.cursor + # substring, line 58 + among_var = self.find_among(RomanianStemmer.a_0, 3) + if among_var == 0: + raise lab2() + # ], line 58 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 59 + # <-, line 59 + if not self.slice_from(u"i"): + return False + elif among_var == 2: + # (, line 60 + # <-, line 60 + if not self.slice_from(u"u"): + return False + elif among_var == 3: + # (, line 61 + # next, line 61 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_RV(self): + if not self.I_pV <= self.cursor: + return False + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_step_0(self): + # (, line 72 + # [, line 73 + self.ket = self.cursor + # substring, line 73 + among_var = self.find_among_b(RomanianStemmer.a_1, 16) + if among_var == 0: + return False + # ], line 73 + self.bra = self.cursor + # call R1, line 73 + if not self.r_R1(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 75 + # delete, line 75 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 77 + # <-, line 77 + if not self.slice_from(u"a"): + return False + elif among_var == 3: + # (, line 79 + # <-, line 79 + if not self.slice_from(u"e"): + return False + elif among_var == 4: + # (, line 81 + # <-, line 81 + if not self.slice_from(u"i"): + return False + elif among_var == 5: + # (, line 83 + # not, line 83 + v_1 = self.limit - self.cursor + try: + # literal, line 83 + if not self.eq_s_b(2, u"ab"): + raise lab0() + return False + except lab0: pass + self.cursor = self.limit - v_1 + # <-, line 83 + if not self.slice_from(u"i"): + return False + elif among_var == 6: + # (, line 85 + # <-, line 85 + if not self.slice_from(u"at"): + return False + elif among_var == 7: + # (, line 87 + # <-, line 87 + if not self.slice_from(u"a\u0163i"): + return False + return True + + def r_combo_suffix(self): + # test, line 91 + v_1 = self.limit - self.cursor + # (, line 91 + # [, line 92 + self.ket = self.cursor + # substring, line 92 + among_var = self.find_among_b(RomanianStemmer.a_2, 46) + if among_var == 0: + return False + # ], line 92 + self.bra = self.cursor + # call R1, line 92 + if not self.r_R1(): + return False + # (, line 92 + if among_var == 0: + return False + elif among_var == 1: + # (, line 100 + # <-, line 101 + if not self.slice_from(u"abil"): + return False + elif among_var == 2: + # (, line 103 + # <-, line 104 + if not self.slice_from(u"ibil"): + return False + elif among_var == 3: + # (, line 106 + # <-, line 107 + if not self.slice_from(u"iv"): + return False + elif among_var == 4: + # (, line 112 + # <-, line 113 + if not self.slice_from(u"ic"): + return False + elif among_var == 5: + # (, line 117 + # <-, line 118 + if not self.slice_from(u"at"): + return False + elif among_var == 6: + # (, line 121 + # <-, line 122 + if not self.slice_from(u"it"): + return False + # set standard_suffix_removed, line 125 + self.B_standard_suffix_removed = True + self.cursor = self.limit - v_1 + return True + + def r_standard_suffix(self): + # (, line 129 + # unset standard_suffix_removed, line 130 + self.B_standard_suffix_removed = False + # repeat, line 131 + try: + while True: + try: + v_1 = self.limit - self.cursor + try: + # call combo_suffix, line 131 + if not self.r_combo_suffix(): + raise lab2() + raise lab1() + except lab2: pass + self.cursor = self.limit - v_1 + raise lab0() + except lab1: pass + except lab0: pass + # [, line 132 + self.ket = self.cursor + # substring, line 132 + among_var = self.find_among_b(RomanianStemmer.a_3, 62) + if among_var == 0: + return False + # ], line 132 + self.bra = self.cursor + # call R2, line 132 + if not self.r_R2(): + return False + # (, line 132 + if among_var == 0: + return False + elif among_var == 1: + # (, line 148 + # delete, line 149 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 151 + # literal, line 152 + if not self.eq_s_b(1, u"\u0163"): + return False + # ], line 152 + self.bra = self.cursor + # <-, line 152 + if not self.slice_from(u"t"): + return False + elif among_var == 3: + # (, line 155 + # <-, line 156 + if not self.slice_from(u"ist"): + return False + # set standard_suffix_removed, line 160 + self.B_standard_suffix_removed = True + return True + + def r_verb_suffix(self): + # setlimit, line 164 + v_1 = self.limit - self.cursor + # tomark, line 164 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 164 + # [, line 165 + self.ket = self.cursor + # substring, line 165 + among_var = self.find_among_b(RomanianStemmer.a_4, 94) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 165 + self.bra = self.cursor + if among_var == 0: + self.limit_backward = v_2 + return False + elif among_var == 1: + # (, line 200 + # or, line 200 + try: + v_3 = self.limit - self.cursor + try: + if not self.out_grouping_b(RomanianStemmer.g_v, 97, 259): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_3 + # literal, line 200 + if not self.eq_s_b(1, u"u"): + self.limit_backward = v_2 + return False + except lab0: pass + # delete, line 200 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 214 + # delete, line 214 + if not self.slice_del(): + return False + + self.limit_backward = v_2 + return True + + def r_vowel_suffix(self): + # (, line 218 + # [, line 219 + self.ket = self.cursor + # substring, line 219 + among_var = self.find_among_b(RomanianStemmer.a_5, 5) + if among_var == 0: + return False + # ], line 219 + self.bra = self.cursor + # call RV, line 219 + if not self.r_RV(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 220 + # delete, line 220 + if not self.slice_del(): + return False + + return True + + def _stem(self): + # (, line 225 + # do, line 226 + v_1 = self.cursor + try: + # call prelude, line 226 + if not self.r_prelude(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # do, line 227 + v_2 = self.cursor + try: + # call mark_regions, line 227 + if not self.r_mark_regions(): + raise lab1() + except lab1: pass + self.cursor = v_2 + # backwards, line 228 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 228 + # do, line 229 + v_3 = self.limit - self.cursor + try: + # call step_0, line 229 + if not self.r_step_0(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 230 + v_4 = self.limit - self.cursor + try: + # call standard_suffix, line 230 + if not self.r_standard_suffix(): + raise lab3() + except lab3: pass + self.cursor = self.limit - v_4 + # do, line 231 + v_5 = self.limit - self.cursor + try: + # (, line 231 + # or, line 231 + try: + v_6 = self.limit - self.cursor + try: + # Boolean test standard_suffix_removed, line 231 + if not self.B_standard_suffix_removed: + raise lab6() + raise lab5() + except lab6: pass + self.cursor = self.limit - v_6 + # call verb_suffix, line 231 + if not self.r_verb_suffix(): + raise lab4() + except lab5: pass + except lab4: pass + self.cursor = self.limit - v_5 + # do, line 232 + v_7 = self.limit - self.cursor + try: + # call vowel_suffix, line 232 + if not self.r_vowel_suffix(): + raise lab7() + except lab7: pass + self.cursor = self.limit - v_7 + self.cursor = self.limit_backward + # do, line 234 + v_8 = self.cursor + try: + # call postlude, line 234 + if not self.r_postlude(): + raise lab8() + except lab8: pass + self.cursor = v_8 + return True + + def equals(self, o): + return isinstance(o, RomanianStemmer) + + def hashCode(self): + return hash("RomanianStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass +class lab16(BaseException): pass +class lab17(BaseException): pass +class lab18(BaseException): pass +class lab19(BaseException): pass +class lab20(BaseException): pass +class lab21(BaseException): pass diff --git a/contrib/snowballstemmer/russian_stemmer.py b/contrib/snowballstemmer/russian_stemmer.py new file mode 100644 index 0000000..3cdb3b6 --- /dev/null +++ b/contrib/snowballstemmer/russian_stemmer.py @@ -0,0 +1,636 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class RussianStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"\u0432", -1, 1), + Among(u"\u0438\u0432", 0, 2), + Among(u"\u044B\u0432", 0, 2), + Among(u"\u0432\u0448\u0438", -1, 1), + Among(u"\u0438\u0432\u0448\u0438", 3, 2), + Among(u"\u044B\u0432\u0448\u0438", 3, 2), + Among(u"\u0432\u0448\u0438\u0441\u044C", -1, 1), + Among(u"\u0438\u0432\u0448\u0438\u0441\u044C", 6, 2), + Among(u"\u044B\u0432\u0448\u0438\u0441\u044C", 6, 2) + ] + + a_1 = [ + Among(u"\u0435\u0435", -1, 1), + Among(u"\u0438\u0435", -1, 1), + Among(u"\u043E\u0435", -1, 1), + Among(u"\u044B\u0435", -1, 1), + Among(u"\u0438\u043C\u0438", -1, 1), + Among(u"\u044B\u043C\u0438", -1, 1), + Among(u"\u0435\u0439", -1, 1), + Among(u"\u0438\u0439", -1, 1), + Among(u"\u043E\u0439", -1, 1), + Among(u"\u044B\u0439", -1, 1), + Among(u"\u0435\u043C", -1, 1), + Among(u"\u0438\u043C", -1, 1), + Among(u"\u043E\u043C", -1, 1), + Among(u"\u044B\u043C", -1, 1), + Among(u"\u0435\u0433\u043E", -1, 1), + Among(u"\u043E\u0433\u043E", -1, 1), + Among(u"\u0435\u043C\u0443", -1, 1), + Among(u"\u043E\u043C\u0443", -1, 1), + Among(u"\u0438\u0445", -1, 1), + Among(u"\u044B\u0445", -1, 1), + Among(u"\u0435\u044E", -1, 1), + Among(u"\u043E\u044E", -1, 1), + Among(u"\u0443\u044E", -1, 1), + Among(u"\u044E\u044E", -1, 1), + Among(u"\u0430\u044F", -1, 1), + Among(u"\u044F\u044F", -1, 1) + ] + + a_2 = [ + Among(u"\u0435\u043C", -1, 1), + Among(u"\u043D\u043D", -1, 1), + Among(u"\u0432\u0448", -1, 1), + Among(u"\u0438\u0432\u0448", 2, 2), + Among(u"\u044B\u0432\u0448", 2, 2), + Among(u"\u0449", -1, 1), + Among(u"\u044E\u0449", 5, 1), + Among(u"\u0443\u044E\u0449", 6, 2) + ] + + a_3 = [ + Among(u"\u0441\u044C", -1, 1), + Among(u"\u0441\u044F", -1, 1) + ] + + a_4 = [ + Among(u"\u043B\u0430", -1, 1), + Among(u"\u0438\u043B\u0430", 0, 2), + Among(u"\u044B\u043B\u0430", 0, 2), + Among(u"\u043D\u0430", -1, 1), + Among(u"\u0435\u043D\u0430", 3, 2), + Among(u"\u0435\u0442\u0435", -1, 1), + Among(u"\u0438\u0442\u0435", -1, 2), + Among(u"\u0439\u0442\u0435", -1, 1), + Among(u"\u0435\u0439\u0442\u0435", 7, 2), + Among(u"\u0443\u0439\u0442\u0435", 7, 2), + Among(u"\u043B\u0438", -1, 1), + Among(u"\u0438\u043B\u0438", 10, 2), + Among(u"\u044B\u043B\u0438", 10, 2), + Among(u"\u0439", -1, 1), + Among(u"\u0435\u0439", 13, 2), + Among(u"\u0443\u0439", 13, 2), + Among(u"\u043B", -1, 1), + Among(u"\u0438\u043B", 16, 2), + Among(u"\u044B\u043B", 16, 2), + Among(u"\u0435\u043C", -1, 1), + Among(u"\u0438\u043C", -1, 2), + Among(u"\u044B\u043C", -1, 2), + Among(u"\u043D", -1, 1), + Among(u"\u0435\u043D", 22, 2), + Among(u"\u043B\u043E", -1, 1), + Among(u"\u0438\u043B\u043E", 24, 2), + Among(u"\u044B\u043B\u043E", 24, 2), + Among(u"\u043D\u043E", -1, 1), + Among(u"\u0435\u043D\u043E", 27, 2), + Among(u"\u043D\u043D\u043E", 27, 1), + Among(u"\u0435\u0442", -1, 1), + Among(u"\u0443\u0435\u0442", 30, 2), + Among(u"\u0438\u0442", -1, 2), + Among(u"\u044B\u0442", -1, 2), + Among(u"\u044E\u0442", -1, 1), + Among(u"\u0443\u044E\u0442", 34, 2), + Among(u"\u044F\u0442", -1, 2), + Among(u"\u043D\u044B", -1, 1), + Among(u"\u0435\u043D\u044B", 37, 2), + Among(u"\u0442\u044C", -1, 1), + Among(u"\u0438\u0442\u044C", 39, 2), + Among(u"\u044B\u0442\u044C", 39, 2), + Among(u"\u0435\u0448\u044C", -1, 1), + Among(u"\u0438\u0448\u044C", -1, 2), + Among(u"\u044E", -1, 2), + Among(u"\u0443\u044E", 44, 2) + ] + + a_5 = [ + Among(u"\u0430", -1, 1), + Among(u"\u0435\u0432", -1, 1), + Among(u"\u043E\u0432", -1, 1), + Among(u"\u0435", -1, 1), + Among(u"\u0438\u0435", 3, 1), + Among(u"\u044C\u0435", 3, 1), + Among(u"\u0438", -1, 1), + Among(u"\u0435\u0438", 6, 1), + Among(u"\u0438\u0438", 6, 1), + Among(u"\u0430\u043C\u0438", 6, 1), + Among(u"\u044F\u043C\u0438", 6, 1), + Among(u"\u0438\u044F\u043C\u0438", 10, 1), + Among(u"\u0439", -1, 1), + Among(u"\u0435\u0439", 12, 1), + Among(u"\u0438\u0435\u0439", 13, 1), + Among(u"\u0438\u0439", 12, 1), + Among(u"\u043E\u0439", 12, 1), + Among(u"\u0430\u043C", -1, 1), + Among(u"\u0435\u043C", -1, 1), + Among(u"\u0438\u0435\u043C", 18, 1), + Among(u"\u043E\u043C", -1, 1), + Among(u"\u044F\u043C", -1, 1), + Among(u"\u0438\u044F\u043C", 21, 1), + Among(u"\u043E", -1, 1), + Among(u"\u0443", -1, 1), + Among(u"\u0430\u0445", -1, 1), + Among(u"\u044F\u0445", -1, 1), + Among(u"\u0438\u044F\u0445", 26, 1), + Among(u"\u044B", -1, 1), + Among(u"\u044C", -1, 1), + Among(u"\u044E", -1, 1), + Among(u"\u0438\u044E", 30, 1), + Among(u"\u044C\u044E", 30, 1), + Among(u"\u044F", -1, 1), + Among(u"\u0438\u044F", 33, 1), + Among(u"\u044C\u044F", 33, 1) + ] + + a_6 = [ + Among(u"\u043E\u0441\u0442", -1, 1), + Among(u"\u043E\u0441\u0442\u044C", -1, 1) + ] + + a_7 = [ + Among(u"\u0435\u0439\u0448\u0435", -1, 1), + Among(u"\u043D", -1, 2), + Among(u"\u0435\u0439\u0448", -1, 1), + Among(u"\u044C", -1, 3) + ] + + g_v = [33, 65, 8, 232] + + I_p2 = 0 + I_pV = 0 + + def copy_from(self, other): + self.I_p2 = other.I_p2 + self.I_pV = other.I_pV + super.copy_from(other) + + + def r_mark_regions(self): + # (, line 57 + self.I_pV = self.limit; + self.I_p2 = self.limit; + # do, line 61 + v_1 = self.cursor + try: + # (, line 61 + # gopast, line 62 + try: + while True: + try: + if not self.in_grouping(RussianStemmer.g_v, 1072, 1103): + raise lab2() + raise lab1() + except lab2: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab1: pass + # setmark pV, line 62 + self.I_pV = self.cursor + # gopast, line 62 + try: + while True: + try: + if not self.out_grouping(RussianStemmer.g_v, 1072, 1103): + raise lab4() + raise lab3() + except lab4: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab3: pass + # gopast, line 63 + try: + while True: + try: + if not self.in_grouping(RussianStemmer.g_v, 1072, 1103): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab5: pass + # gopast, line 63 + try: + while True: + try: + if not self.out_grouping(RussianStemmer.g_v, 1072, 1103): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab7: pass + # setmark p2, line 63 + self.I_p2 = self.cursor + except lab0: pass + self.cursor = v_1 + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_perfective_gerund(self): + # (, line 71 + # [, line 72 + self.ket = self.cursor + # substring, line 72 + among_var = self.find_among_b(RussianStemmer.a_0, 9) + if among_var == 0: + return False + # ], line 72 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 76 + # or, line 76 + try: + v_1 = self.limit - self.cursor + try: + # literal, line 76 + if not self.eq_s_b(1, u"\u0430"): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # literal, line 76 + if not self.eq_s_b(1, u"\u044F"): + return False + except lab0: pass + # delete, line 76 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 83 + # delete, line 83 + if not self.slice_del(): + return False + + return True + + def r_adjective(self): + # (, line 87 + # [, line 88 + self.ket = self.cursor + # substring, line 88 + among_var = self.find_among_b(RussianStemmer.a_1, 26) + if among_var == 0: + return False + # ], line 88 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 97 + # delete, line 97 + if not self.slice_del(): + return False + + return True + + def r_adjectival(self): + # (, line 101 + # call adjective, line 102 + if not self.r_adjective(): + return False + # try, line 109 + v_1 = self.limit - self.cursor + try: + # (, line 109 + # [, line 110 + self.ket = self.cursor + # substring, line 110 + among_var = self.find_among_b(RussianStemmer.a_2, 8) + if among_var == 0: + self.cursor = self.limit - v_1 + raise lab0() + # ], line 110 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_1 + raise lab0() + elif among_var == 1: + # (, line 115 + # or, line 115 + try: + v_2 = self.limit - self.cursor + try: + # literal, line 115 + if not self.eq_s_b(1, u"\u0430"): + raise lab2() + raise lab1() + except lab2: pass + self.cursor = self.limit - v_2 + # literal, line 115 + if not self.eq_s_b(1, u"\u044F"): + self.cursor = self.limit - v_1 + raise lab0() + except lab1: pass + # delete, line 115 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 122 + # delete, line 122 + if not self.slice_del(): + return False + + except lab0: pass + return True + + def r_reflexive(self): + # (, line 128 + # [, line 129 + self.ket = self.cursor + # substring, line 129 + among_var = self.find_among_b(RussianStemmer.a_3, 2) + if among_var == 0: + return False + # ], line 129 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 132 + # delete, line 132 + if not self.slice_del(): + return False + + return True + + def r_verb(self): + # (, line 136 + # [, line 137 + self.ket = self.cursor + # substring, line 137 + among_var = self.find_among_b(RussianStemmer.a_4, 46) + if among_var == 0: + return False + # ], line 137 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 143 + # or, line 143 + try: + v_1 = self.limit - self.cursor + try: + # literal, line 143 + if not self.eq_s_b(1, u"\u0430"): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # literal, line 143 + if not self.eq_s_b(1, u"\u044F"): + return False + except lab0: pass + # delete, line 143 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 151 + # delete, line 151 + if not self.slice_del(): + return False + + return True + + def r_noun(self): + # (, line 159 + # [, line 160 + self.ket = self.cursor + # substring, line 160 + among_var = self.find_among_b(RussianStemmer.a_5, 36) + if among_var == 0: + return False + # ], line 160 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 167 + # delete, line 167 + if not self.slice_del(): + return False + + return True + + def r_derivational(self): + # (, line 175 + # [, line 176 + self.ket = self.cursor + # substring, line 176 + among_var = self.find_among_b(RussianStemmer.a_6, 2) + if among_var == 0: + return False + # ], line 176 + self.bra = self.cursor + # call R2, line 176 + if not self.r_R2(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 179 + # delete, line 179 + if not self.slice_del(): + return False + + return True + + def r_tidy_up(self): + # (, line 183 + # [, line 184 + self.ket = self.cursor + # substring, line 184 + among_var = self.find_among_b(RussianStemmer.a_7, 4) + if among_var == 0: + return False + # ], line 184 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 188 + # delete, line 188 + if not self.slice_del(): + return False + + # [, line 189 + self.ket = self.cursor + # literal, line 189 + if not self.eq_s_b(1, u"\u043D"): + return False + # ], line 189 + self.bra = self.cursor + # literal, line 189 + if not self.eq_s_b(1, u"\u043D"): + return False + # delete, line 189 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 192 + # literal, line 192 + if not self.eq_s_b(1, u"\u043D"): + return False + # delete, line 192 + if not self.slice_del(): + return False + + elif among_var == 3: + # (, line 194 + # delete, line 194 + if not self.slice_del(): + return False + + return True + + def _stem(self): + # (, line 199 + # do, line 201 + v_1 = self.cursor + try: + # call mark_regions, line 201 + if not self.r_mark_regions(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # backwards, line 202 + self.limit_backward = self.cursor + self.cursor = self.limit + # setlimit, line 202 + v_2 = self.limit - self.cursor + # tomark, line 202 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_3 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_2 + # (, line 202 + # do, line 203 + v_4 = self.limit - self.cursor + try: + # (, line 203 + # or, line 204 + try: + v_5 = self.limit - self.cursor + try: + # call perfective_gerund, line 204 + if not self.r_perfective_gerund(): + raise lab3() + raise lab2() + except lab3: pass + self.cursor = self.limit - v_5 + # (, line 205 + # try, line 205 + v_6 = self.limit - self.cursor + try: + # call reflexive, line 205 + if not self.r_reflexive(): + self.cursor = self.limit - v_6 + raise lab4() + except lab4: pass + # or, line 206 + try: + v_7 = self.limit - self.cursor + try: + # call adjectival, line 206 + if not self.r_adjectival(): + raise lab6() + raise lab5() + except lab6: pass + self.cursor = self.limit - v_7 + try: + # call verb, line 206 + if not self.r_verb(): + raise lab7() + raise lab5() + except lab7: pass + self.cursor = self.limit - v_7 + # call noun, line 206 + if not self.r_noun(): + raise lab1() + except lab5: pass + except lab2: pass + except lab1: pass + self.cursor = self.limit - v_4 + # try, line 209 + v_8 = self.limit - self.cursor + try: + # (, line 209 + # [, line 209 + self.ket = self.cursor + # literal, line 209 + if not self.eq_s_b(1, u"\u0438"): + self.cursor = self.limit - v_8 + raise lab8() + # ], line 209 + self.bra = self.cursor + # delete, line 209 + if not self.slice_del(): + return False + + except lab8: pass + # do, line 212 + v_9 = self.limit - self.cursor + try: + # call derivational, line 212 + if not self.r_derivational(): + raise lab9() + except lab9: pass + self.cursor = self.limit - v_9 + # do, line 213 + v_10 = self.limit - self.cursor + try: + # call tidy_up, line 213 + if not self.r_tidy_up(): + raise lab10() + except lab10: pass + self.cursor = self.limit - v_10 + self.limit_backward = v_3 + self.cursor = self.limit_backward + return True + + def equals(self, o): + return isinstance(o, RussianStemmer) + + def hashCode(self): + return hash("RussianStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass diff --git a/contrib/snowballstemmer/spanish_stemmer.py b/contrib/snowballstemmer/spanish_stemmer.py new file mode 100644 index 0000000..b9d325c --- /dev/null +++ b/contrib/snowballstemmer/spanish_stemmer.py @@ -0,0 +1,1032 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class SpanishStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"", -1, 6), + Among(u"\u00E1", 0, 1), + Among(u"\u00E9", 0, 2), + Among(u"\u00ED", 0, 3), + Among(u"\u00F3", 0, 4), + Among(u"\u00FA", 0, 5) + ] + + a_1 = [ + Among(u"la", -1, -1), + Among(u"sela", 0, -1), + Among(u"le", -1, -1), + Among(u"me", -1, -1), + Among(u"se", -1, -1), + Among(u"lo", -1, -1), + Among(u"selo", 5, -1), + Among(u"las", -1, -1), + Among(u"selas", 7, -1), + Among(u"les", -1, -1), + Among(u"los", -1, -1), + Among(u"selos", 10, -1), + Among(u"nos", -1, -1) + ] + + a_2 = [ + Among(u"ando", -1, 6), + Among(u"iendo", -1, 6), + Among(u"yendo", -1, 7), + Among(u"\u00E1ndo", -1, 2), + Among(u"i\u00E9ndo", -1, 1), + Among(u"ar", -1, 6), + Among(u"er", -1, 6), + Among(u"ir", -1, 6), + Among(u"\u00E1r", -1, 3), + Among(u"\u00E9r", -1, 4), + Among(u"\u00EDr", -1, 5) + ] + + a_3 = [ + Among(u"ic", -1, -1), + Among(u"ad", -1, -1), + Among(u"os", -1, -1), + Among(u"iv", -1, 1) + ] + + a_4 = [ + Among(u"able", -1, 1), + Among(u"ible", -1, 1), + Among(u"ante", -1, 1) + ] + + a_5 = [ + Among(u"ic", -1, 1), + Among(u"abil", -1, 1), + Among(u"iv", -1, 1) + ] + + a_6 = [ + Among(u"ica", -1, 1), + Among(u"ancia", -1, 2), + Among(u"encia", -1, 5), + Among(u"adora", -1, 2), + Among(u"osa", -1, 1), + Among(u"ista", -1, 1), + Among(u"iva", -1, 9), + Among(u"anza", -1, 1), + Among(u"log\u00EDa", -1, 3), + Among(u"idad", -1, 8), + Among(u"able", -1, 1), + Among(u"ible", -1, 1), + Among(u"ante", -1, 2), + Among(u"mente", -1, 7), + Among(u"amente", 13, 6), + Among(u"aci\u00F3n", -1, 2), + Among(u"uci\u00F3n", -1, 4), + Among(u"ico", -1, 1), + Among(u"ismo", -1, 1), + Among(u"oso", -1, 1), + Among(u"amiento", -1, 1), + Among(u"imiento", -1, 1), + Among(u"ivo", -1, 9), + Among(u"ador", -1, 2), + Among(u"icas", -1, 1), + Among(u"ancias", -1, 2), + Among(u"encias", -1, 5), + Among(u"adoras", -1, 2), + Among(u"osas", -1, 1), + Among(u"istas", -1, 1), + Among(u"ivas", -1, 9), + Among(u"anzas", -1, 1), + Among(u"log\u00EDas", -1, 3), + Among(u"idades", -1, 8), + Among(u"ables", -1, 1), + Among(u"ibles", -1, 1), + Among(u"aciones", -1, 2), + Among(u"uciones", -1, 4), + Among(u"adores", -1, 2), + Among(u"antes", -1, 2), + Among(u"icos", -1, 1), + Among(u"ismos", -1, 1), + Among(u"osos", -1, 1), + Among(u"amientos", -1, 1), + Among(u"imientos", -1, 1), + Among(u"ivos", -1, 9) + ] + + a_7 = [ + Among(u"ya", -1, 1), + Among(u"ye", -1, 1), + Among(u"yan", -1, 1), + Among(u"yen", -1, 1), + Among(u"yeron", -1, 1), + Among(u"yendo", -1, 1), + Among(u"yo", -1, 1), + Among(u"yas", -1, 1), + Among(u"yes", -1, 1), + Among(u"yais", -1, 1), + Among(u"yamos", -1, 1), + Among(u"y\u00F3", -1, 1) + ] + + a_8 = [ + Among(u"aba", -1, 2), + Among(u"ada", -1, 2), + Among(u"ida", -1, 2), + Among(u"ara", -1, 2), + Among(u"iera", -1, 2), + Among(u"\u00EDa", -1, 2), + Among(u"ar\u00EDa", 5, 2), + Among(u"er\u00EDa", 5, 2), + Among(u"ir\u00EDa", 5, 2), + Among(u"ad", -1, 2), + Among(u"ed", -1, 2), + Among(u"id", -1, 2), + Among(u"ase", -1, 2), + Among(u"iese", -1, 2), + Among(u"aste", -1, 2), + Among(u"iste", -1, 2), + Among(u"an", -1, 2), + Among(u"aban", 16, 2), + Among(u"aran", 16, 2), + Among(u"ieran", 16, 2), + Among(u"\u00EDan", 16, 2), + Among(u"ar\u00EDan", 20, 2), + Among(u"er\u00EDan", 20, 2), + Among(u"ir\u00EDan", 20, 2), + Among(u"en", -1, 1), + Among(u"asen", 24, 2), + Among(u"iesen", 24, 2), + Among(u"aron", -1, 2), + Among(u"ieron", -1, 2), + Among(u"ar\u00E1n", -1, 2), + Among(u"er\u00E1n", -1, 2), + Among(u"ir\u00E1n", -1, 2), + Among(u"ado", -1, 2), + Among(u"ido", -1, 2), + Among(u"ando", -1, 2), + Among(u"iendo", -1, 2), + Among(u"ar", -1, 2), + Among(u"er", -1, 2), + Among(u"ir", -1, 2), + Among(u"as", -1, 2), + Among(u"abas", 39, 2), + Among(u"adas", 39, 2), + Among(u"idas", 39, 2), + Among(u"aras", 39, 2), + Among(u"ieras", 39, 2), + Among(u"\u00EDas", 39, 2), + Among(u"ar\u00EDas", 45, 2), + Among(u"er\u00EDas", 45, 2), + Among(u"ir\u00EDas", 45, 2), + Among(u"es", -1, 1), + Among(u"ases", 49, 2), + Among(u"ieses", 49, 2), + Among(u"abais", -1, 2), + Among(u"arais", -1, 2), + Among(u"ierais", -1, 2), + Among(u"\u00EDais", -1, 2), + Among(u"ar\u00EDais", 55, 2), + Among(u"er\u00EDais", 55, 2), + Among(u"ir\u00EDais", 55, 2), + Among(u"aseis", -1, 2), + Among(u"ieseis", -1, 2), + Among(u"asteis", -1, 2), + Among(u"isteis", -1, 2), + Among(u"\u00E1is", -1, 2), + Among(u"\u00E9is", -1, 1), + Among(u"ar\u00E9is", 64, 2), + Among(u"er\u00E9is", 64, 2), + Among(u"ir\u00E9is", 64, 2), + Among(u"ados", -1, 2), + Among(u"idos", -1, 2), + Among(u"amos", -1, 2), + Among(u"\u00E1bamos", 70, 2), + Among(u"\u00E1ramos", 70, 2), + Among(u"i\u00E9ramos", 70, 2), + Among(u"\u00EDamos", 70, 2), + Among(u"ar\u00EDamos", 74, 2), + Among(u"er\u00EDamos", 74, 2), + Among(u"ir\u00EDamos", 74, 2), + Among(u"emos", -1, 1), + Among(u"aremos", 78, 2), + Among(u"eremos", 78, 2), + Among(u"iremos", 78, 2), + Among(u"\u00E1semos", 78, 2), + Among(u"i\u00E9semos", 78, 2), + Among(u"imos", -1, 2), + Among(u"ar\u00E1s", -1, 2), + Among(u"er\u00E1s", -1, 2), + Among(u"ir\u00E1s", -1, 2), + Among(u"\u00EDs", -1, 2), + Among(u"ar\u00E1", -1, 2), + Among(u"er\u00E1", -1, 2), + Among(u"ir\u00E1", -1, 2), + Among(u"ar\u00E9", -1, 2), + Among(u"er\u00E9", -1, 2), + Among(u"ir\u00E9", -1, 2), + Among(u"i\u00F3", -1, 2) + ] + + a_9 = [ + Among(u"a", -1, 1), + Among(u"e", -1, 2), + Among(u"o", -1, 1), + Among(u"os", -1, 1), + Among(u"\u00E1", -1, 1), + Among(u"\u00E9", -1, 2), + Among(u"\u00ED", -1, 1), + Among(u"\u00F3", -1, 1) + ] + + g_v = [17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 4, 10] + + I_p2 = 0 + I_p1 = 0 + I_pV = 0 + + def copy_from(self, other): + self.I_p2 = other.I_p2 + self.I_p1 = other.I_p1 + self.I_pV = other.I_pV + super.copy_from(other) + + + def r_mark_regions(self): + # (, line 31 + self.I_pV = self.limit; + self.I_p1 = self.limit; + self.I_p2 = self.limit; + # do, line 37 + v_1 = self.cursor + try: + # (, line 37 + # or, line 39 + try: + v_2 = self.cursor + try: + # (, line 38 + if not self.in_grouping(SpanishStemmer.g_v, 97, 252): + raise lab2() + # or, line 38 + try: + v_3 = self.cursor + try: + # (, line 38 + if not self.out_grouping(SpanishStemmer.g_v, 97, 252): + raise lab4() + # gopast, line 38 + try: + while True: + try: + if not self.in_grouping(SpanishStemmer.g_v, 97, 252): + raise lab6() + raise lab5() + except lab6: pass + if self.cursor >= self.limit: + raise lab4() + self.cursor += 1 + except lab5: pass + raise lab3() + except lab4: pass + self.cursor = v_3 + # (, line 38 + if not self.in_grouping(SpanishStemmer.g_v, 97, 252): + raise lab2() + # gopast, line 38 + try: + while True: + try: + if not self.out_grouping(SpanishStemmer.g_v, 97, 252): + raise lab8() + raise lab7() + except lab8: pass + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab7: pass + except lab3: pass + raise lab1() + except lab2: pass + self.cursor = v_2 + # (, line 40 + if not self.out_grouping(SpanishStemmer.g_v, 97, 252): + raise lab0() + # or, line 40 + try: + v_6 = self.cursor + try: + # (, line 40 + if not self.out_grouping(SpanishStemmer.g_v, 97, 252): + raise lab10() + # gopast, line 40 + try: + while True: + try: + if not self.in_grouping(SpanishStemmer.g_v, 97, 252): + raise lab12() + raise lab11() + except lab12: pass + if self.cursor >= self.limit: + raise lab10() + self.cursor += 1 + except lab11: pass + raise lab9() + except lab10: pass + self.cursor = v_6 + # (, line 40 + if not self.in_grouping(SpanishStemmer.g_v, 97, 252): + raise lab0() + # next, line 40 + if self.cursor >= self.limit: + raise lab0() + self.cursor += 1 + except lab9: pass + except lab1: pass + # setmark pV, line 41 + self.I_pV = self.cursor + except lab0: pass + self.cursor = v_1 + # do, line 43 + v_8 = self.cursor + try: + # (, line 43 + # gopast, line 44 + try: + while True: + try: + if not self.in_grouping(SpanishStemmer.g_v, 97, 252): + raise lab15() + raise lab14() + except lab15: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab14: pass + # gopast, line 44 + try: + while True: + try: + if not self.out_grouping(SpanishStemmer.g_v, 97, 252): + raise lab17() + raise lab16() + except lab17: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab16: pass + # setmark p1, line 44 + self.I_p1 = self.cursor + # gopast, line 45 + try: + while True: + try: + if not self.in_grouping(SpanishStemmer.g_v, 97, 252): + raise lab19() + raise lab18() + except lab19: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab18: pass + # gopast, line 45 + try: + while True: + try: + if not self.out_grouping(SpanishStemmer.g_v, 97, 252): + raise lab21() + raise lab20() + except lab21: pass + if self.cursor >= self.limit: + raise lab13() + self.cursor += 1 + except lab20: pass + # setmark p2, line 45 + self.I_p2 = self.cursor + except lab13: pass + self.cursor = v_8 + return True + + def r_postlude(self): + # repeat, line 49 + try: + while True: + try: + v_1 = self.cursor + try: + # (, line 49 + # [, line 50 + self.bra = self.cursor + # substring, line 50 + among_var = self.find_among(SpanishStemmer.a_0, 6) + if among_var == 0: + raise lab2() + # ], line 50 + self.ket = self.cursor + if among_var == 0: + raise lab2() + elif among_var == 1: + # (, line 51 + # <-, line 51 + if not self.slice_from(u"a"): + return False + elif among_var == 2: + # (, line 52 + # <-, line 52 + if not self.slice_from(u"e"): + return False + elif among_var == 3: + # (, line 53 + # <-, line 53 + if not self.slice_from(u"i"): + return False + elif among_var == 4: + # (, line 54 + # <-, line 54 + if not self.slice_from(u"o"): + return False + elif among_var == 5: + # (, line 55 + # <-, line 55 + if not self.slice_from(u"u"): + return False + elif among_var == 6: + # (, line 57 + # next, line 57 + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + raise lab1() + except lab2: pass + self.cursor = v_1 + raise lab0() + except lab1: pass + except lab0: pass + return True + + def r_RV(self): + if not self.I_pV <= self.cursor: + return False + return True + + def r_R1(self): + if not self.I_p1 <= self.cursor: + return False + return True + + def r_R2(self): + if not self.I_p2 <= self.cursor: + return False + return True + + def r_attached_pronoun(self): + # (, line 67 + # [, line 68 + self.ket = self.cursor + # substring, line 68 + if self.find_among_b(SpanishStemmer.a_1, 13) == 0: + return False + # ], line 68 + self.bra = self.cursor + # substring, line 72 + among_var = self.find_among_b(SpanishStemmer.a_2, 11) + if among_var == 0: + return False + # call RV, line 72 + if not self.r_RV(): + return False + if among_var == 0: + return False + elif among_var == 1: + # (, line 73 + # ], line 73 + self.bra = self.cursor + # <-, line 73 + if not self.slice_from(u"iendo"): + return False + elif among_var == 2: + # (, line 74 + # ], line 74 + self.bra = self.cursor + # <-, line 74 + if not self.slice_from(u"ando"): + return False + elif among_var == 3: + # (, line 75 + # ], line 75 + self.bra = self.cursor + # <-, line 75 + if not self.slice_from(u"ar"): + return False + elif among_var == 4: + # (, line 76 + # ], line 76 + self.bra = self.cursor + # <-, line 76 + if not self.slice_from(u"er"): + return False + elif among_var == 5: + # (, line 77 + # ], line 77 + self.bra = self.cursor + # <-, line 77 + if not self.slice_from(u"ir"): + return False + elif among_var == 6: + # (, line 81 + # delete, line 81 + if not self.slice_del(): + return False + + elif among_var == 7: + # (, line 82 + # literal, line 82 + if not self.eq_s_b(1, u"u"): + return False + # delete, line 82 + if not self.slice_del(): + return False + + return True + + def r_standard_suffix(self): + # (, line 86 + # [, line 87 + self.ket = self.cursor + # substring, line 87 + among_var = self.find_among_b(SpanishStemmer.a_6, 46) + if among_var == 0: + return False + # ], line 87 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 98 + # call R2, line 99 + if not self.r_R2(): + return False + # delete, line 99 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 104 + # call R2, line 105 + if not self.r_R2(): + return False + # delete, line 105 + if not self.slice_del(): + return False + + # try, line 106 + v_1 = self.limit - self.cursor + try: + # (, line 106 + # [, line 106 + self.ket = self.cursor + # literal, line 106 + if not self.eq_s_b(2, u"ic"): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 106 + self.bra = self.cursor + # call R2, line 106 + if not self.r_R2(): + self.cursor = self.limit - v_1 + raise lab0() + # delete, line 106 + if not self.slice_del(): + return False + + except lab0: pass + elif among_var == 3: + # (, line 110 + # call R2, line 111 + if not self.r_R2(): + return False + # <-, line 111 + if not self.slice_from(u"log"): + return False + elif among_var == 4: + # (, line 114 + # call R2, line 115 + if not self.r_R2(): + return False + # <-, line 115 + if not self.slice_from(u"u"): + return False + elif among_var == 5: + # (, line 118 + # call R2, line 119 + if not self.r_R2(): + return False + # <-, line 119 + if not self.slice_from(u"ente"): + return False + elif among_var == 6: + # (, line 122 + # call R1, line 123 + if not self.r_R1(): + return False + # delete, line 123 + if not self.slice_del(): + return False + + # try, line 124 + v_2 = self.limit - self.cursor + try: + # (, line 124 + # [, line 125 + self.ket = self.cursor + # substring, line 125 + among_var = self.find_among_b(SpanishStemmer.a_3, 4) + if among_var == 0: + self.cursor = self.limit - v_2 + raise lab1() + # ], line 125 + self.bra = self.cursor + # call R2, line 125 + if not self.r_R2(): + self.cursor = self.limit - v_2 + raise lab1() + # delete, line 125 + if not self.slice_del(): + return False + + if among_var == 0: + self.cursor = self.limit - v_2 + raise lab1() + elif among_var == 1: + # (, line 126 + # [, line 126 + self.ket = self.cursor + # literal, line 126 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_2 + raise lab1() + # ], line 126 + self.bra = self.cursor + # call R2, line 126 + if not self.r_R2(): + self.cursor = self.limit - v_2 + raise lab1() + # delete, line 126 + if not self.slice_del(): + return False + + except lab1: pass + elif among_var == 7: + # (, line 134 + # call R2, line 135 + if not self.r_R2(): + return False + # delete, line 135 + if not self.slice_del(): + return False + + # try, line 136 + v_3 = self.limit - self.cursor + try: + # (, line 136 + # [, line 137 + self.ket = self.cursor + # substring, line 137 + among_var = self.find_among_b(SpanishStemmer.a_4, 3) + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab2() + # ], line 137 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_3 + raise lab2() + elif among_var == 1: + # (, line 140 + # call R2, line 140 + if not self.r_R2(): + self.cursor = self.limit - v_3 + raise lab2() + # delete, line 140 + if not self.slice_del(): + return False + + except lab2: pass + elif among_var == 8: + # (, line 146 + # call R2, line 147 + if not self.r_R2(): + return False + # delete, line 147 + if not self.slice_del(): + return False + + # try, line 148 + v_4 = self.limit - self.cursor + try: + # (, line 148 + # [, line 149 + self.ket = self.cursor + # substring, line 149 + among_var = self.find_among_b(SpanishStemmer.a_5, 3) + if among_var == 0: + self.cursor = self.limit - v_4 + raise lab3() + # ], line 149 + self.bra = self.cursor + if among_var == 0: + self.cursor = self.limit - v_4 + raise lab3() + elif among_var == 1: + # (, line 152 + # call R2, line 152 + if not self.r_R2(): + self.cursor = self.limit - v_4 + raise lab3() + # delete, line 152 + if not self.slice_del(): + return False + + except lab3: pass + elif among_var == 9: + # (, line 158 + # call R2, line 159 + if not self.r_R2(): + return False + # delete, line 159 + if not self.slice_del(): + return False + + # try, line 160 + v_5 = self.limit - self.cursor + try: + # (, line 160 + # [, line 161 + self.ket = self.cursor + # literal, line 161 + if not self.eq_s_b(2, u"at"): + self.cursor = self.limit - v_5 + raise lab4() + # ], line 161 + self.bra = self.cursor + # call R2, line 161 + if not self.r_R2(): + self.cursor = self.limit - v_5 + raise lab4() + # delete, line 161 + if not self.slice_del(): + return False + + except lab4: pass + return True + + def r_y_verb_suffix(self): + # (, line 167 + # setlimit, line 168 + v_1 = self.limit - self.cursor + # tomark, line 168 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 168 + # [, line 168 + self.ket = self.cursor + # substring, line 168 + among_var = self.find_among_b(SpanishStemmer.a_7, 12) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 168 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 171 + # literal, line 171 + if not self.eq_s_b(1, u"u"): + return False + # delete, line 171 + if not self.slice_del(): + return False + + return True + + def r_verb_suffix(self): + # (, line 175 + # setlimit, line 176 + v_1 = self.limit - self.cursor + # tomark, line 176 + if self.cursor < self.I_pV: + return False + self.cursor = self.I_pV + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 176 + # [, line 176 + self.ket = self.cursor + # substring, line 176 + among_var = self.find_among_b(SpanishStemmer.a_8, 96) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 176 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 179 + # try, line 179 + v_3 = self.limit - self.cursor + try: + # (, line 179 + # literal, line 179 + if not self.eq_s_b(1, u"u"): + self.cursor = self.limit - v_3 + raise lab0() + # test, line 179 + v_4 = self.limit - self.cursor + # literal, line 179 + if not self.eq_s_b(1, u"g"): + self.cursor = self.limit - v_3 + raise lab0() + self.cursor = self.limit - v_4 + except lab0: pass + # ], line 179 + self.bra = self.cursor + # delete, line 179 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 200 + # delete, line 200 + if not self.slice_del(): + return False + + return True + + def r_residual_suffix(self): + # (, line 204 + # [, line 205 + self.ket = self.cursor + # substring, line 205 + among_var = self.find_among_b(SpanishStemmer.a_9, 8) + if among_var == 0: + return False + # ], line 205 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 208 + # call RV, line 208 + if not self.r_RV(): + return False + # delete, line 208 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 210 + # call RV, line 210 + if not self.r_RV(): + return False + # delete, line 210 + if not self.slice_del(): + return False + + # try, line 210 + v_1 = self.limit - self.cursor + try: + # (, line 210 + # [, line 210 + self.ket = self.cursor + # literal, line 210 + if not self.eq_s_b(1, u"u"): + self.cursor = self.limit - v_1 + raise lab0() + # ], line 210 + self.bra = self.cursor + # test, line 210 + v_2 = self.limit - self.cursor + # literal, line 210 + if not self.eq_s_b(1, u"g"): + self.cursor = self.limit - v_1 + raise lab0() + self.cursor = self.limit - v_2 + # call RV, line 210 + if not self.r_RV(): + self.cursor = self.limit - v_1 + raise lab0() + # delete, line 210 + if not self.slice_del(): + return False + + except lab0: pass + return True + + def _stem(self): + # (, line 215 + # do, line 216 + v_1 = self.cursor + try: + # call mark_regions, line 216 + if not self.r_mark_regions(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # backwards, line 217 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 217 + # do, line 218 + v_2 = self.limit - self.cursor + try: + # call attached_pronoun, line 218 + if not self.r_attached_pronoun(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 219 + v_3 = self.limit - self.cursor + try: + # (, line 219 + # or, line 219 + try: + v_4 = self.limit - self.cursor + try: + # call standard_suffix, line 219 + if not self.r_standard_suffix(): + raise lab4() + raise lab3() + except lab4: pass + self.cursor = self.limit - v_4 + try: + # call y_verb_suffix, line 220 + if not self.r_y_verb_suffix(): + raise lab5() + raise lab3() + except lab5: pass + self.cursor = self.limit - v_4 + # call verb_suffix, line 221 + if not self.r_verb_suffix(): + raise lab2() + except lab3: pass + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 223 + v_5 = self.limit - self.cursor + try: + # call residual_suffix, line 223 + if not self.r_residual_suffix(): + raise lab6() + except lab6: pass + self.cursor = self.limit - v_5 + self.cursor = self.limit_backward + # do, line 225 + v_6 = self.cursor + try: + # call postlude, line 225 + if not self.r_postlude(): + raise lab7() + except lab7: pass + self.cursor = v_6 + return True + + def equals(self, o): + return isinstance(o, SpanishStemmer) + + def hashCode(self): + return hash("SpanishStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass +class lab16(BaseException): pass +class lab17(BaseException): pass +class lab18(BaseException): pass +class lab19(BaseException): pass +class lab20(BaseException): pass +class lab21(BaseException): pass diff --git a/contrib/snowballstemmer/swedish_stemmer.py b/contrib/snowballstemmer/swedish_stemmer.py new file mode 100644 index 0000000..d0d1c6f --- /dev/null +++ b/contrib/snowballstemmer/swedish_stemmer.py @@ -0,0 +1,304 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class SwedishStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"a", -1, 1), + Among(u"arna", 0, 1), + Among(u"erna", 0, 1), + Among(u"heterna", 2, 1), + Among(u"orna", 0, 1), + Among(u"ad", -1, 1), + Among(u"e", -1, 1), + Among(u"ade", 6, 1), + Among(u"ande", 6, 1), + Among(u"arne", 6, 1), + Among(u"are", 6, 1), + Among(u"aste", 6, 1), + Among(u"en", -1, 1), + Among(u"anden", 12, 1), + Among(u"aren", 12, 1), + Among(u"heten", 12, 1), + Among(u"ern", -1, 1), + Among(u"ar", -1, 1), + Among(u"er", -1, 1), + Among(u"heter", 18, 1), + Among(u"or", -1, 1), + Among(u"s", -1, 2), + Among(u"as", 21, 1), + Among(u"arnas", 22, 1), + Among(u"ernas", 22, 1), + Among(u"ornas", 22, 1), + Among(u"es", 21, 1), + Among(u"ades", 26, 1), + Among(u"andes", 26, 1), + Among(u"ens", 21, 1), + Among(u"arens", 29, 1), + Among(u"hetens", 29, 1), + Among(u"erns", 21, 1), + Among(u"at", -1, 1), + Among(u"andet", -1, 1), + Among(u"het", -1, 1), + Among(u"ast", -1, 1) + ] + + a_1 = [ + Among(u"dd", -1, -1), + Among(u"gd", -1, -1), + Among(u"nn", -1, -1), + Among(u"dt", -1, -1), + Among(u"gt", -1, -1), + Among(u"kt", -1, -1), + Among(u"tt", -1, -1) + ] + + a_2 = [ + Among(u"ig", -1, 1), + Among(u"lig", 0, 1), + Among(u"els", -1, 1), + Among(u"fullt", -1, 3), + Among(u"l\u00F6st", -1, 2) + ] + + g_v = [17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 32] + + g_s_ending = [119, 127, 149] + + I_x = 0 + I_p1 = 0 + + def copy_from(self, other): + self.I_x = other.I_x + self.I_p1 = other.I_p1 + super.copy_from(other) + + + def r_mark_regions(self): + # (, line 26 + self.I_p1 = self.limit; + # test, line 29 + v_1 = self.cursor + # (, line 29 + # hop, line 29 + c = self.cursor + 3 + if 0 > c or c > self.limit: + return False + self.cursor = c + # setmark x, line 29 + self.I_x = self.cursor + self.cursor = v_1 + # goto, line 30 + try: + while True: + v_2 = self.cursor + try: + if not self.in_grouping(SwedishStemmer.g_v, 97, 246): + raise lab1() + self.cursor = v_2 + raise lab0() + except lab1: pass + self.cursor = v_2 + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab0: pass + # gopast, line 30 + try: + while True: + try: + if not self.out_grouping(SwedishStemmer.g_v, 97, 246): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab2: pass + # setmark p1, line 30 + self.I_p1 = self.cursor + # try, line 31 + try: + # (, line 31 + if not (self.I_p1 < self.I_x): + raise lab4() + self.I_p1 = self.I_x; + except lab4: pass + return True + + def r_main_suffix(self): + # (, line 36 + # setlimit, line 37 + v_1 = self.limit - self.cursor + # tomark, line 37 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 37 + # [, line 37 + self.ket = self.cursor + # substring, line 37 + among_var = self.find_among_b(SwedishStemmer.a_0, 37) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 37 + self.bra = self.cursor + self.limit_backward = v_2 + if among_var == 0: + return False + elif among_var == 1: + # (, line 44 + # delete, line 44 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 46 + if not self.in_grouping_b(SwedishStemmer.g_s_ending, 98, 121): + return False + # delete, line 46 + if not self.slice_del(): + return False + + return True + + def r_consonant_pair(self): + # setlimit, line 50 + v_1 = self.limit - self.cursor + # tomark, line 50 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 50 + # and, line 52 + v_3 = self.limit - self.cursor + # among, line 51 + if self.find_among_b(SwedishStemmer.a_1, 7) == 0: + self.limit_backward = v_2 + return False + self.cursor = self.limit - v_3 + # (, line 52 + # [, line 52 + self.ket = self.cursor + # next, line 52 + if self.cursor <= self.limit_backward: + self.limit_backward = v_2 + return False + self.cursor -= 1 + # ], line 52 + self.bra = self.cursor + # delete, line 52 + if not self.slice_del(): + return False + + self.limit_backward = v_2 + return True + + def r_other_suffix(self): + # setlimit, line 55 + v_1 = self.limit - self.cursor + # tomark, line 55 + if self.cursor < self.I_p1: + return False + self.cursor = self.I_p1 + v_2 = self.limit_backward + self.limit_backward = self.cursor + self.cursor = self.limit - v_1 + # (, line 55 + # [, line 56 + self.ket = self.cursor + # substring, line 56 + among_var = self.find_among_b(SwedishStemmer.a_2, 5) + if among_var == 0: + self.limit_backward = v_2 + return False + # ], line 56 + self.bra = self.cursor + if among_var == 0: + self.limit_backward = v_2 + return False + elif among_var == 1: + # (, line 57 + # delete, line 57 + if not self.slice_del(): + return False + + elif among_var == 2: + # (, line 58 + # <-, line 58 + if not self.slice_from(u"l\u00F6s"): + return False + elif among_var == 3: + # (, line 59 + # <-, line 59 + if not self.slice_from(u"full"): + return False + self.limit_backward = v_2 + return True + + def _stem(self): + # (, line 64 + # do, line 66 + v_1 = self.cursor + try: + # call mark_regions, line 66 + if not self.r_mark_regions(): + raise lab0() + except lab0: pass + self.cursor = v_1 + # backwards, line 67 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 67 + # do, line 68 + v_2 = self.limit - self.cursor + try: + # call main_suffix, line 68 + if not self.r_main_suffix(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 69 + v_3 = self.limit - self.cursor + try: + # call consonant_pair, line 69 + if not self.r_consonant_pair(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + # do, line 70 + v_4 = self.limit - self.cursor + try: + # call other_suffix, line 70 + if not self.r_other_suffix(): + raise lab3() + except lab3: pass + self.cursor = self.limit - v_4 + self.cursor = self.limit_backward + return True + + def equals(self, o): + return isinstance(o, SwedishStemmer) + + def hashCode(self): + return hash("SwedishStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass diff --git a/contrib/snowballstemmer/turkish_stemmer.py b/contrib/snowballstemmer/turkish_stemmer.py new file mode 100644 index 0000000..4776bd0 --- /dev/null +++ b/contrib/snowballstemmer/turkish_stemmer.py @@ -0,0 +1,2601 @@ +# self file was generated automatically by the Snowball to Python interpreter + +from .basestemmer import BaseStemmer +from .among import Among + + +class TurkishStemmer(BaseStemmer): + ''' + self class was automatically generated by a Snowball to Python interpreter + It implements the stemming algorithm defined by a snowball script. + ''' + serialVersionUID = 1 + + a_0 = [ + Among(u"m", -1, -1), + Among(u"n", -1, -1), + Among(u"miz", -1, -1), + Among(u"niz", -1, -1), + Among(u"muz", -1, -1), + Among(u"nuz", -1, -1), + Among(u"m\u00FCz", -1, -1), + Among(u"n\u00FCz", -1, -1), + Among(u"m\u0131z", -1, -1), + Among(u"n\u0131z", -1, -1) + ] + + a_1 = [ + Among(u"leri", -1, -1), + Among(u"lar\u0131", -1, -1) + ] + + a_2 = [ + Among(u"ni", -1, -1), + Among(u"nu", -1, -1), + Among(u"n\u00FC", -1, -1), + Among(u"n\u0131", -1, -1) + ] + + a_3 = [ + Among(u"in", -1, -1), + Among(u"un", -1, -1), + Among(u"\u00FCn", -1, -1), + Among(u"\u0131n", -1, -1) + ] + + a_4 = [ + Among(u"a", -1, -1), + Among(u"e", -1, -1) + ] + + a_5 = [ + Among(u"na", -1, -1), + Among(u"ne", -1, -1) + ] + + a_6 = [ + Among(u"da", -1, -1), + Among(u"ta", -1, -1), + Among(u"de", -1, -1), + Among(u"te", -1, -1) + ] + + a_7 = [ + Among(u"nda", -1, -1), + Among(u"nde", -1, -1) + ] + + a_8 = [ + Among(u"dan", -1, -1), + Among(u"tan", -1, -1), + Among(u"den", -1, -1), + Among(u"ten", -1, -1) + ] + + a_9 = [ + Among(u"ndan", -1, -1), + Among(u"nden", -1, -1) + ] + + a_10 = [ + Among(u"la", -1, -1), + Among(u"le", -1, -1) + ] + + a_11 = [ + Among(u"ca", -1, -1), + Among(u"ce", -1, -1) + ] + + a_12 = [ + Among(u"im", -1, -1), + Among(u"um", -1, -1), + Among(u"\u00FCm", -1, -1), + Among(u"\u0131m", -1, -1) + ] + + a_13 = [ + Among(u"sin", -1, -1), + Among(u"sun", -1, -1), + Among(u"s\u00FCn", -1, -1), + Among(u"s\u0131n", -1, -1) + ] + + a_14 = [ + Among(u"iz", -1, -1), + Among(u"uz", -1, -1), + Among(u"\u00FCz", -1, -1), + Among(u"\u0131z", -1, -1) + ] + + a_15 = [ + Among(u"siniz", -1, -1), + Among(u"sunuz", -1, -1), + Among(u"s\u00FCn\u00FCz", -1, -1), + Among(u"s\u0131n\u0131z", -1, -1) + ] + + a_16 = [ + Among(u"lar", -1, -1), + Among(u"ler", -1, -1) + ] + + a_17 = [ + Among(u"niz", -1, -1), + Among(u"nuz", -1, -1), + Among(u"n\u00FCz", -1, -1), + Among(u"n\u0131z", -1, -1) + ] + + a_18 = [ + Among(u"dir", -1, -1), + Among(u"tir", -1, -1), + Among(u"dur", -1, -1), + Among(u"tur", -1, -1), + Among(u"d\u00FCr", -1, -1), + Among(u"t\u00FCr", -1, -1), + Among(u"d\u0131r", -1, -1), + Among(u"t\u0131r", -1, -1) + ] + + a_19 = [ + Among(u"cas\u0131na", -1, -1), + Among(u"cesine", -1, -1) + ] + + a_20 = [ + Among(u"di", -1, -1), + Among(u"ti", -1, -1), + Among(u"dik", -1, -1), + Among(u"tik", -1, -1), + Among(u"duk", -1, -1), + Among(u"tuk", -1, -1), + Among(u"d\u00FCk", -1, -1), + Among(u"t\u00FCk", -1, -1), + Among(u"d\u0131k", -1, -1), + Among(u"t\u0131k", -1, -1), + Among(u"dim", -1, -1), + Among(u"tim", -1, -1), + Among(u"dum", -1, -1), + Among(u"tum", -1, -1), + Among(u"d\u00FCm", -1, -1), + Among(u"t\u00FCm", -1, -1), + Among(u"d\u0131m", -1, -1), + Among(u"t\u0131m", -1, -1), + Among(u"din", -1, -1), + Among(u"tin", -1, -1), + Among(u"dun", -1, -1), + Among(u"tun", -1, -1), + Among(u"d\u00FCn", -1, -1), + Among(u"t\u00FCn", -1, -1), + Among(u"d\u0131n", -1, -1), + Among(u"t\u0131n", -1, -1), + Among(u"du", -1, -1), + Among(u"tu", -1, -1), + Among(u"d\u00FC", -1, -1), + Among(u"t\u00FC", -1, -1), + Among(u"d\u0131", -1, -1), + Among(u"t\u0131", -1, -1) + ] + + a_21 = [ + Among(u"sa", -1, -1), + Among(u"se", -1, -1), + Among(u"sak", -1, -1), + Among(u"sek", -1, -1), + Among(u"sam", -1, -1), + Among(u"sem", -1, -1), + Among(u"san", -1, -1), + Among(u"sen", -1, -1) + ] + + a_22 = [ + Among(u"mi\u015F", -1, -1), + Among(u"mu\u015F", -1, -1), + Among(u"m\u00FC\u015F", -1, -1), + Among(u"m\u0131\u015F", -1, -1) + ] + + a_23 = [ + Among(u"b", -1, 1), + Among(u"c", -1, 2), + Among(u"d", -1, 3), + Among(u"\u011F", -1, 4) + ] + + g_vowel = [17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 0, 0, 0, 0, 0, 0, 1] + + g_U = [1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1] + + g_vowel1 = [1, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + + g_vowel2 = [17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130] + + g_vowel3 = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + + g_vowel4 = [17] + + g_vowel5 = [65] + + g_vowel6 = [65] + + B_continue_stemming_noun_suffixes = False + I_strlen = 0 + + def copy_from(self, other): + self.B_continue_stemming_noun_suffixes = other.B_continue_stemming_noun_suffixes + self.I_strlen = other.I_strlen + super.copy_from(other) + + + def r_check_vowel_harmony(self): + # (, line 111 + # test, line 112 + v_1 = self.limit - self.cursor + # (, line 113 + # (, line 114 + # goto, line 114 + try: + while True: + v_2 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab1() + self.cursor = self.limit - v_2 + raise lab0() + except lab1: pass + self.cursor = self.limit - v_2 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab0: pass + # (, line 115 + # or, line 116 + try: + v_3 = self.limit - self.cursor + try: + # (, line 116 + # literal, line 116 + if not self.eq_s_b(1, u"a"): + raise lab3() + # goto, line 116 + try: + while True: + v_4 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel1, 97, 305): + raise lab5() + self.cursor = self.limit - v_4 + raise lab4() + except lab5: pass + self.cursor = self.limit - v_4 + if self.cursor <= self.limit_backward: + raise lab3() + self.cursor -= 1 + except lab4: pass + raise lab2() + except lab3: pass + self.cursor = self.limit - v_3 + try: + # (, line 117 + # literal, line 117 + if not self.eq_s_b(1, u"e"): + raise lab6() + # goto, line 117 + try: + while True: + v_5 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel2, 101, 252): + raise lab8() + self.cursor = self.limit - v_5 + raise lab7() + except lab8: pass + self.cursor = self.limit - v_5 + if self.cursor <= self.limit_backward: + raise lab6() + self.cursor -= 1 + except lab7: pass + raise lab2() + except lab6: pass + self.cursor = self.limit - v_3 + try: + # (, line 118 + # literal, line 118 + if not self.eq_s_b(1, u"\u0131"): + raise lab9() + # goto, line 118 + try: + while True: + v_6 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel3, 97, 305): + raise lab11() + self.cursor = self.limit - v_6 + raise lab10() + except lab11: pass + self.cursor = self.limit - v_6 + if self.cursor <= self.limit_backward: + raise lab9() + self.cursor -= 1 + except lab10: pass + raise lab2() + except lab9: pass + self.cursor = self.limit - v_3 + try: + # (, line 119 + # literal, line 119 + if not self.eq_s_b(1, u"i"): + raise lab12() + # goto, line 119 + try: + while True: + v_7 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel4, 101, 105): + raise lab14() + self.cursor = self.limit - v_7 + raise lab13() + except lab14: pass + self.cursor = self.limit - v_7 + if self.cursor <= self.limit_backward: + raise lab12() + self.cursor -= 1 + except lab13: pass + raise lab2() + except lab12: pass + self.cursor = self.limit - v_3 + try: + # (, line 120 + # literal, line 120 + if not self.eq_s_b(1, u"o"): + raise lab15() + # goto, line 120 + try: + while True: + v_8 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel5, 111, 117): + raise lab17() + self.cursor = self.limit - v_8 + raise lab16() + except lab17: pass + self.cursor = self.limit - v_8 + if self.cursor <= self.limit_backward: + raise lab15() + self.cursor -= 1 + except lab16: pass + raise lab2() + except lab15: pass + self.cursor = self.limit - v_3 + try: + # (, line 121 + # literal, line 121 + if not self.eq_s_b(1, u"\u00F6"): + raise lab18() + # goto, line 121 + try: + while True: + v_9 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel6, 246, 252): + raise lab20() + self.cursor = self.limit - v_9 + raise lab19() + except lab20: pass + self.cursor = self.limit - v_9 + if self.cursor <= self.limit_backward: + raise lab18() + self.cursor -= 1 + except lab19: pass + raise lab2() + except lab18: pass + self.cursor = self.limit - v_3 + try: + # (, line 122 + # literal, line 122 + if not self.eq_s_b(1, u"u"): + raise lab21() + # goto, line 122 + try: + while True: + v_10 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel5, 111, 117): + raise lab23() + self.cursor = self.limit - v_10 + raise lab22() + except lab23: pass + self.cursor = self.limit - v_10 + if self.cursor <= self.limit_backward: + raise lab21() + self.cursor -= 1 + except lab22: pass + raise lab2() + except lab21: pass + self.cursor = self.limit - v_3 + # (, line 123 + # literal, line 123 + if not self.eq_s_b(1, u"\u00FC"): + return False + # goto, line 123 + try: + while True: + v_11 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel6, 246, 252): + raise lab25() + self.cursor = self.limit - v_11 + raise lab24() + except lab25: pass + self.cursor = self.limit - v_11 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab24: pass + except lab2: pass + self.cursor = self.limit - v_1 + return True + + def r_mark_suffix_with_optional_n_consonant(self): + # (, line 132 + # or, line 134 + try: + v_1 = self.limit - self.cursor + try: + # (, line 133 + # (, line 133 + # test, line 133 + v_2 = self.limit - self.cursor + # literal, line 133 + if not self.eq_s_b(1, u"n"): + raise lab1() + self.cursor = self.limit - v_2 + # next, line 133 + if self.cursor <= self.limit_backward: + raise lab1() + self.cursor -= 1 + # (, line 133 + # test, line 133 + v_3 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab1() + self.cursor = self.limit - v_3 + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 135 + # (, line 135 + # not, line 135 + v_4 = self.limit - self.cursor + try: + # (, line 135 + # test, line 135 + v_5 = self.limit - self.cursor + # literal, line 135 + if not self.eq_s_b(1, u"n"): + raise lab2() + self.cursor = self.limit - v_5 + return False + except lab2: pass + self.cursor = self.limit - v_4 + # test, line 135 + v_6 = self.limit - self.cursor + # (, line 135 + # next, line 135 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # (, line 135 + # test, line 135 + v_7 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + return False + self.cursor = self.limit - v_7 + self.cursor = self.limit - v_6 + except lab0: pass + return True + + def r_mark_suffix_with_optional_s_consonant(self): + # (, line 143 + # or, line 145 + try: + v_1 = self.limit - self.cursor + try: + # (, line 144 + # (, line 144 + # test, line 144 + v_2 = self.limit - self.cursor + # literal, line 144 + if not self.eq_s_b(1, u"s"): + raise lab1() + self.cursor = self.limit - v_2 + # next, line 144 + if self.cursor <= self.limit_backward: + raise lab1() + self.cursor -= 1 + # (, line 144 + # test, line 144 + v_3 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab1() + self.cursor = self.limit - v_3 + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 146 + # (, line 146 + # not, line 146 + v_4 = self.limit - self.cursor + try: + # (, line 146 + # test, line 146 + v_5 = self.limit - self.cursor + # literal, line 146 + if not self.eq_s_b(1, u"s"): + raise lab2() + self.cursor = self.limit - v_5 + return False + except lab2: pass + self.cursor = self.limit - v_4 + # test, line 146 + v_6 = self.limit - self.cursor + # (, line 146 + # next, line 146 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # (, line 146 + # test, line 146 + v_7 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + return False + self.cursor = self.limit - v_7 + self.cursor = self.limit - v_6 + except lab0: pass + return True + + def r_mark_suffix_with_optional_y_consonant(self): + # (, line 153 + # or, line 155 + try: + v_1 = self.limit - self.cursor + try: + # (, line 154 + # (, line 154 + # test, line 154 + v_2 = self.limit - self.cursor + # literal, line 154 + if not self.eq_s_b(1, u"y"): + raise lab1() + self.cursor = self.limit - v_2 + # next, line 154 + if self.cursor <= self.limit_backward: + raise lab1() + self.cursor -= 1 + # (, line 154 + # test, line 154 + v_3 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab1() + self.cursor = self.limit - v_3 + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 156 + # (, line 156 + # not, line 156 + v_4 = self.limit - self.cursor + try: + # (, line 156 + # test, line 156 + v_5 = self.limit - self.cursor + # literal, line 156 + if not self.eq_s_b(1, u"y"): + raise lab2() + self.cursor = self.limit - v_5 + return False + except lab2: pass + self.cursor = self.limit - v_4 + # test, line 156 + v_6 = self.limit - self.cursor + # (, line 156 + # next, line 156 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # (, line 156 + # test, line 156 + v_7 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + return False + self.cursor = self.limit - v_7 + self.cursor = self.limit - v_6 + except lab0: pass + return True + + def r_mark_suffix_with_optional_U_vowel(self): + # (, line 159 + # or, line 161 + try: + v_1 = self.limit - self.cursor + try: + # (, line 160 + # (, line 160 + # test, line 160 + v_2 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_U, 105, 305): + raise lab1() + self.cursor = self.limit - v_2 + # next, line 160 + if self.cursor <= self.limit_backward: + raise lab1() + self.cursor -= 1 + # (, line 160 + # test, line 160 + v_3 = self.limit - self.cursor + if not self.out_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab1() + self.cursor = self.limit - v_3 + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + # (, line 162 + # (, line 162 + # not, line 162 + v_4 = self.limit - self.cursor + try: + # (, line 162 + # test, line 162 + v_5 = self.limit - self.cursor + if not self.in_grouping_b(TurkishStemmer.g_U, 105, 305): + raise lab2() + self.cursor = self.limit - v_5 + return False + except lab2: pass + self.cursor = self.limit - v_4 + # test, line 162 + v_6 = self.limit - self.cursor + # (, line 162 + # next, line 162 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + # (, line 162 + # test, line 162 + v_7 = self.limit - self.cursor + if not self.out_grouping_b(TurkishStemmer.g_vowel, 97, 305): + return False + self.cursor = self.limit - v_7 + self.cursor = self.limit - v_6 + except lab0: pass + return True + + def r_mark_possessives(self): + # (, line 166 + # among, line 167 + if self.find_among_b(TurkishStemmer.a_0, 10) == 0: + return False + # (, line 169 + # call mark_suffix_with_optional_U_vowel, line 169 + if not self.r_mark_suffix_with_optional_U_vowel(): + return False + return True + + def r_mark_sU(self): + # (, line 172 + # call check_vowel_harmony, line 173 + if not self.r_check_vowel_harmony(): + return False + if not self.in_grouping_b(TurkishStemmer.g_U, 105, 305): + return False + # (, line 175 + # call mark_suffix_with_optional_s_consonant, line 175 + if not self.r_mark_suffix_with_optional_s_consonant(): + return False + return True + + def r_mark_lArI(self): + # (, line 178 + # among, line 179 + if self.find_among_b(TurkishStemmer.a_1, 2) == 0: + return False + return True + + def r_mark_yU(self): + # (, line 182 + # call check_vowel_harmony, line 183 + if not self.r_check_vowel_harmony(): + return False + if not self.in_grouping_b(TurkishStemmer.g_U, 105, 305): + return False + # (, line 185 + # call mark_suffix_with_optional_y_consonant, line 185 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_nU(self): + # (, line 188 + # call check_vowel_harmony, line 189 + if not self.r_check_vowel_harmony(): + return False + # among, line 190 + if self.find_among_b(TurkishStemmer.a_2, 4) == 0: + return False + return True + + def r_mark_nUn(self): + # (, line 193 + # call check_vowel_harmony, line 194 + if not self.r_check_vowel_harmony(): + return False + # among, line 195 + if self.find_among_b(TurkishStemmer.a_3, 4) == 0: + return False + # (, line 196 + # call mark_suffix_with_optional_n_consonant, line 196 + if not self.r_mark_suffix_with_optional_n_consonant(): + return False + return True + + def r_mark_yA(self): + # (, line 199 + # call check_vowel_harmony, line 200 + if not self.r_check_vowel_harmony(): + return False + # among, line 201 + if self.find_among_b(TurkishStemmer.a_4, 2) == 0: + return False + # (, line 202 + # call mark_suffix_with_optional_y_consonant, line 202 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_nA(self): + # (, line 205 + # call check_vowel_harmony, line 206 + if not self.r_check_vowel_harmony(): + return False + # among, line 207 + if self.find_among_b(TurkishStemmer.a_5, 2) == 0: + return False + return True + + def r_mark_DA(self): + # (, line 210 + # call check_vowel_harmony, line 211 + if not self.r_check_vowel_harmony(): + return False + # among, line 212 + if self.find_among_b(TurkishStemmer.a_6, 4) == 0: + return False + return True + + def r_mark_ndA(self): + # (, line 215 + # call check_vowel_harmony, line 216 + if not self.r_check_vowel_harmony(): + return False + # among, line 217 + if self.find_among_b(TurkishStemmer.a_7, 2) == 0: + return False + return True + + def r_mark_DAn(self): + # (, line 220 + # call check_vowel_harmony, line 221 + if not self.r_check_vowel_harmony(): + return False + # among, line 222 + if self.find_among_b(TurkishStemmer.a_8, 4) == 0: + return False + return True + + def r_mark_ndAn(self): + # (, line 225 + # call check_vowel_harmony, line 226 + if not self.r_check_vowel_harmony(): + return False + # among, line 227 + if self.find_among_b(TurkishStemmer.a_9, 2) == 0: + return False + return True + + def r_mark_ylA(self): + # (, line 230 + # call check_vowel_harmony, line 231 + if not self.r_check_vowel_harmony(): + return False + # among, line 232 + if self.find_among_b(TurkishStemmer.a_10, 2) == 0: + return False + # (, line 233 + # call mark_suffix_with_optional_y_consonant, line 233 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_ki(self): + # (, line 236 + # literal, line 237 + if not self.eq_s_b(2, u"ki"): + return False + return True + + def r_mark_ncA(self): + # (, line 240 + # call check_vowel_harmony, line 241 + if not self.r_check_vowel_harmony(): + return False + # among, line 242 + if self.find_among_b(TurkishStemmer.a_11, 2) == 0: + return False + # (, line 243 + # call mark_suffix_with_optional_n_consonant, line 243 + if not self.r_mark_suffix_with_optional_n_consonant(): + return False + return True + + def r_mark_yUm(self): + # (, line 246 + # call check_vowel_harmony, line 247 + if not self.r_check_vowel_harmony(): + return False + # among, line 248 + if self.find_among_b(TurkishStemmer.a_12, 4) == 0: + return False + # (, line 249 + # call mark_suffix_with_optional_y_consonant, line 249 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_sUn(self): + # (, line 252 + # call check_vowel_harmony, line 253 + if not self.r_check_vowel_harmony(): + return False + # among, line 254 + if self.find_among_b(TurkishStemmer.a_13, 4) == 0: + return False + return True + + def r_mark_yUz(self): + # (, line 257 + # call check_vowel_harmony, line 258 + if not self.r_check_vowel_harmony(): + return False + # among, line 259 + if self.find_among_b(TurkishStemmer.a_14, 4) == 0: + return False + # (, line 260 + # call mark_suffix_with_optional_y_consonant, line 260 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_sUnUz(self): + # (, line 263 + # among, line 264 + if self.find_among_b(TurkishStemmer.a_15, 4) == 0: + return False + return True + + def r_mark_lAr(self): + # (, line 267 + # call check_vowel_harmony, line 268 + if not self.r_check_vowel_harmony(): + return False + # among, line 269 + if self.find_among_b(TurkishStemmer.a_16, 2) == 0: + return False + return True + + def r_mark_nUz(self): + # (, line 272 + # call check_vowel_harmony, line 273 + if not self.r_check_vowel_harmony(): + return False + # among, line 274 + if self.find_among_b(TurkishStemmer.a_17, 4) == 0: + return False + return True + + def r_mark_DUr(self): + # (, line 277 + # call check_vowel_harmony, line 278 + if not self.r_check_vowel_harmony(): + return False + # among, line 279 + if self.find_among_b(TurkishStemmer.a_18, 8) == 0: + return False + return True + + def r_mark_cAsInA(self): + # (, line 282 + # among, line 283 + if self.find_among_b(TurkishStemmer.a_19, 2) == 0: + return False + return True + + def r_mark_yDU(self): + # (, line 286 + # call check_vowel_harmony, line 287 + if not self.r_check_vowel_harmony(): + return False + # among, line 288 + if self.find_among_b(TurkishStemmer.a_20, 32) == 0: + return False + # (, line 292 + # call mark_suffix_with_optional_y_consonant, line 292 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_ysA(self): + # (, line 296 + # among, line 297 + if self.find_among_b(TurkishStemmer.a_21, 8) == 0: + return False + # (, line 298 + # call mark_suffix_with_optional_y_consonant, line 298 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_ymUs_(self): + # (, line 301 + # call check_vowel_harmony, line 302 + if not self.r_check_vowel_harmony(): + return False + # among, line 303 + if self.find_among_b(TurkishStemmer.a_22, 4) == 0: + return False + # (, line 304 + # call mark_suffix_with_optional_y_consonant, line 304 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_mark_yken(self): + # (, line 307 + # literal, line 308 + if not self.eq_s_b(3, u"ken"): + return False + # (, line 308 + # call mark_suffix_with_optional_y_consonant, line 308 + if not self.r_mark_suffix_with_optional_y_consonant(): + return False + return True + + def r_stem_nominal_verb_suffixes(self): + # (, line 311 + # [, line 312 + self.ket = self.cursor + # set continue_stemming_noun_suffixes, line 313 + self.B_continue_stemming_noun_suffixes = True + # or, line 315 + try: + v_1 = self.limit - self.cursor + try: + # (, line 314 + # or, line 314 + try: + v_2 = self.limit - self.cursor + try: + # call mark_ymUs_, line 314 + if not self.r_mark_ymUs_(): + raise lab3() + raise lab2() + except lab3: pass + self.cursor = self.limit - v_2 + try: + # call mark_yDU, line 314 + if not self.r_mark_yDU(): + raise lab4() + raise lab2() + except lab4: pass + self.cursor = self.limit - v_2 + try: + # call mark_ysA, line 314 + if not self.r_mark_ysA(): + raise lab5() + raise lab2() + except lab5: pass + self.cursor = self.limit - v_2 + # call mark_yken, line 314 + if not self.r_mark_yken(): + raise lab1() + except lab2: pass + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + try: + # (, line 316 + # call mark_cAsInA, line 316 + if not self.r_mark_cAsInA(): + raise lab6() + # (, line 316 + # or, line 316 + try: + v_3 = self.limit - self.cursor + try: + # call mark_sUnUz, line 316 + if not self.r_mark_sUnUz(): + raise lab8() + raise lab7() + except lab8: pass + self.cursor = self.limit - v_3 + try: + # call mark_lAr, line 316 + if not self.r_mark_lAr(): + raise lab9() + raise lab7() + except lab9: pass + self.cursor = self.limit - v_3 + try: + # call mark_yUm, line 316 + if not self.r_mark_yUm(): + raise lab10() + raise lab7() + except lab10: pass + self.cursor = self.limit - v_3 + try: + # call mark_sUn, line 316 + if not self.r_mark_sUn(): + raise lab11() + raise lab7() + except lab11: pass + self.cursor = self.limit - v_3 + try: + # call mark_yUz, line 316 + if not self.r_mark_yUz(): + raise lab12() + raise lab7() + except lab12: pass + self.cursor = self.limit - v_3 + except lab7: pass + # call mark_ymUs_, line 316 + if not self.r_mark_ymUs_(): + raise lab6() + raise lab0() + except lab6: pass + self.cursor = self.limit - v_1 + try: + # (, line 318 + # call mark_lAr, line 319 + if not self.r_mark_lAr(): + raise lab13() + # ], line 319 + self.bra = self.cursor + # delete, line 319 + if not self.slice_del(): + return False + + # try, line 319 + v_4 = self.limit - self.cursor + try: + # (, line 319 + # [, line 319 + self.ket = self.cursor + # (, line 319 + # or, line 319 + try: + v_5 = self.limit - self.cursor + try: + # call mark_DUr, line 319 + if not self.r_mark_DUr(): + raise lab16() + raise lab15() + except lab16: pass + self.cursor = self.limit - v_5 + try: + # call mark_yDU, line 319 + if not self.r_mark_yDU(): + raise lab17() + raise lab15() + except lab17: pass + self.cursor = self.limit - v_5 + try: + # call mark_ysA, line 319 + if not self.r_mark_ysA(): + raise lab18() + raise lab15() + except lab18: pass + self.cursor = self.limit - v_5 + # call mark_ymUs_, line 319 + if not self.r_mark_ymUs_(): + self.cursor = self.limit - v_4 + raise lab14() + except lab15: pass + except lab14: pass + # unset continue_stemming_noun_suffixes, line 320 + self.B_continue_stemming_noun_suffixes = False + raise lab0() + except lab13: pass + self.cursor = self.limit - v_1 + try: + # (, line 323 + # call mark_nUz, line 323 + if not self.r_mark_nUz(): + raise lab19() + # (, line 323 + # or, line 323 + try: + v_6 = self.limit - self.cursor + try: + # call mark_yDU, line 323 + if not self.r_mark_yDU(): + raise lab21() + raise lab20() + except lab21: pass + self.cursor = self.limit - v_6 + # call mark_ysA, line 323 + if not self.r_mark_ysA(): + raise lab19() + except lab20: pass + raise lab0() + except lab19: pass + self.cursor = self.limit - v_1 + try: + # (, line 325 + # (, line 325 + # or, line 325 + try: + v_7 = self.limit - self.cursor + try: + # call mark_sUnUz, line 325 + if not self.r_mark_sUnUz(): + raise lab24() + raise lab23() + except lab24: pass + self.cursor = self.limit - v_7 + try: + # call mark_yUz, line 325 + if not self.r_mark_yUz(): + raise lab25() + raise lab23() + except lab25: pass + self.cursor = self.limit - v_7 + try: + # call mark_sUn, line 325 + if not self.r_mark_sUn(): + raise lab26() + raise lab23() + except lab26: pass + self.cursor = self.limit - v_7 + # call mark_yUm, line 325 + if not self.r_mark_yUm(): + raise lab22() + except lab23: pass + # ], line 325 + self.bra = self.cursor + # delete, line 325 + if not self.slice_del(): + return False + + # try, line 325 + v_8 = self.limit - self.cursor + try: + # (, line 325 + # [, line 325 + self.ket = self.cursor + # call mark_ymUs_, line 325 + if not self.r_mark_ymUs_(): + self.cursor = self.limit - v_8 + raise lab27() + except lab27: pass + raise lab0() + except lab22: pass + self.cursor = self.limit - v_1 + # (, line 327 + # call mark_DUr, line 327 + if not self.r_mark_DUr(): + return False + # ], line 327 + self.bra = self.cursor + # delete, line 327 + if not self.slice_del(): + return False + + # try, line 327 + v_9 = self.limit - self.cursor + try: + # (, line 327 + # [, line 327 + self.ket = self.cursor + # (, line 327 + # or, line 327 + try: + v_10 = self.limit - self.cursor + try: + # call mark_sUnUz, line 327 + if not self.r_mark_sUnUz(): + raise lab30() + raise lab29() + except lab30: pass + self.cursor = self.limit - v_10 + try: + # call mark_lAr, line 327 + if not self.r_mark_lAr(): + raise lab31() + raise lab29() + except lab31: pass + self.cursor = self.limit - v_10 + try: + # call mark_yUm, line 327 + if not self.r_mark_yUm(): + raise lab32() + raise lab29() + except lab32: pass + self.cursor = self.limit - v_10 + try: + # call mark_sUn, line 327 + if not self.r_mark_sUn(): + raise lab33() + raise lab29() + except lab33: pass + self.cursor = self.limit - v_10 + try: + # call mark_yUz, line 327 + if not self.r_mark_yUz(): + raise lab34() + raise lab29() + except lab34: pass + self.cursor = self.limit - v_10 + except lab29: pass + # call mark_ymUs_, line 327 + if not self.r_mark_ymUs_(): + self.cursor = self.limit - v_9 + raise lab28() + except lab28: pass + except lab0: pass + # ], line 328 + self.bra = self.cursor + # delete, line 328 + if not self.slice_del(): + return False + + return True + + def r_stem_suffix_chain_before_ki(self): + # (, line 332 + # [, line 333 + self.ket = self.cursor + # call mark_ki, line 334 + if not self.r_mark_ki(): + return False + # (, line 335 + # or, line 342 + try: + v_1 = self.limit - self.cursor + try: + # (, line 336 + # call mark_DA, line 336 + if not self.r_mark_DA(): + raise lab1() + # ], line 336 + self.bra = self.cursor + # delete, line 336 + if not self.slice_del(): + return False + + # try, line 336 + v_2 = self.limit - self.cursor + try: + # (, line 336 + # [, line 336 + self.ket = self.cursor + # or, line 338 + try: + v_3 = self.limit - self.cursor + try: + # (, line 337 + # call mark_lAr, line 337 + if not self.r_mark_lAr(): + raise lab4() + # ], line 337 + self.bra = self.cursor + # delete, line 337 + if not self.slice_del(): + return False + + # try, line 337 + v_4 = self.limit - self.cursor + try: + # (, line 337 + # call stem_suffix_chain_before_ki, line 337 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_4 + raise lab5() + except lab5: pass + raise lab3() + except lab4: pass + self.cursor = self.limit - v_3 + # (, line 339 + # call mark_possessives, line 339 + if not self.r_mark_possessives(): + self.cursor = self.limit - v_2 + raise lab2() + # ], line 339 + self.bra = self.cursor + # delete, line 339 + if not self.slice_del(): + return False + + # try, line 339 + v_5 = self.limit - self.cursor + try: + # (, line 339 + # [, line 339 + self.ket = self.cursor + # call mark_lAr, line 339 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_5 + raise lab6() + # ], line 339 + self.bra = self.cursor + # delete, line 339 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 339 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_5 + raise lab6() + except lab6: pass + except lab3: pass + except lab2: pass + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + try: + # (, line 343 + # call mark_nUn, line 343 + if not self.r_mark_nUn(): + raise lab7() + # ], line 343 + self.bra = self.cursor + # delete, line 343 + if not self.slice_del(): + return False + + # try, line 343 + v_6 = self.limit - self.cursor + try: + # (, line 343 + # [, line 343 + self.ket = self.cursor + # or, line 345 + try: + v_7 = self.limit - self.cursor + try: + # (, line 344 + # call mark_lArI, line 344 + if not self.r_mark_lArI(): + raise lab10() + # ], line 344 + self.bra = self.cursor + # delete, line 344 + if not self.slice_del(): + return False + + raise lab9() + except lab10: pass + self.cursor = self.limit - v_7 + try: + # (, line 346 + # [, line 346 + self.ket = self.cursor + # or, line 346 + try: + v_8 = self.limit - self.cursor + try: + # call mark_possessives, line 346 + if not self.r_mark_possessives(): + raise lab13() + raise lab12() + except lab13: pass + self.cursor = self.limit - v_8 + # call mark_sU, line 346 + if not self.r_mark_sU(): + raise lab11() + except lab12: pass + # ], line 346 + self.bra = self.cursor + # delete, line 346 + if not self.slice_del(): + return False + + # try, line 346 + v_9 = self.limit - self.cursor + try: + # (, line 346 + # [, line 346 + self.ket = self.cursor + # call mark_lAr, line 346 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_9 + raise lab14() + # ], line 346 + self.bra = self.cursor + # delete, line 346 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 346 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_9 + raise lab14() + except lab14: pass + raise lab9() + except lab11: pass + self.cursor = self.limit - v_7 + # (, line 348 + # call stem_suffix_chain_before_ki, line 348 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_6 + raise lab8() + except lab9: pass + except lab8: pass + raise lab0() + except lab7: pass + self.cursor = self.limit - v_1 + # (, line 351 + # call mark_ndA, line 351 + if not self.r_mark_ndA(): + return False + # (, line 351 + # or, line 353 + try: + v_10 = self.limit - self.cursor + try: + # (, line 352 + # call mark_lArI, line 352 + if not self.r_mark_lArI(): + raise lab16() + # ], line 352 + self.bra = self.cursor + # delete, line 352 + if not self.slice_del(): + return False + + raise lab15() + except lab16: pass + self.cursor = self.limit - v_10 + try: + # (, line 354 + # (, line 354 + # call mark_sU, line 354 + if not self.r_mark_sU(): + raise lab17() + # ], line 354 + self.bra = self.cursor + # delete, line 354 + if not self.slice_del(): + return False + + # try, line 354 + v_11 = self.limit - self.cursor + try: + # (, line 354 + # [, line 354 + self.ket = self.cursor + # call mark_lAr, line 354 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_11 + raise lab18() + # ], line 354 + self.bra = self.cursor + # delete, line 354 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 354 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_11 + raise lab18() + except lab18: pass + raise lab15() + except lab17: pass + self.cursor = self.limit - v_10 + # (, line 356 + # call stem_suffix_chain_before_ki, line 356 + if not self.r_stem_suffix_chain_before_ki(): + return False + except lab15: pass + except lab0: pass + return True + + def r_stem_noun_suffixes(self): + # (, line 361 + # or, line 363 + try: + v_1 = self.limit - self.cursor + try: + # (, line 362 + # [, line 362 + self.ket = self.cursor + # call mark_lAr, line 362 + if not self.r_mark_lAr(): + raise lab1() + # ], line 362 + self.bra = self.cursor + # delete, line 362 + if not self.slice_del(): + return False + + # try, line 362 + v_2 = self.limit - self.cursor + try: + # (, line 362 + # call stem_suffix_chain_before_ki, line 362 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_2 + raise lab2() + except lab2: pass + raise lab0() + except lab1: pass + self.cursor = self.limit - v_1 + try: + # (, line 364 + # [, line 364 + self.ket = self.cursor + # call mark_ncA, line 364 + if not self.r_mark_ncA(): + raise lab3() + # ], line 364 + self.bra = self.cursor + # delete, line 364 + if not self.slice_del(): + return False + + # try, line 365 + v_3 = self.limit - self.cursor + try: + # (, line 365 + # or, line 367 + try: + v_4 = self.limit - self.cursor + try: + # (, line 366 + # [, line 366 + self.ket = self.cursor + # call mark_lArI, line 366 + if not self.r_mark_lArI(): + raise lab6() + # ], line 366 + self.bra = self.cursor + # delete, line 366 + if not self.slice_del(): + return False + + raise lab5() + except lab6: pass + self.cursor = self.limit - v_4 + try: + # (, line 368 + # [, line 368 + self.ket = self.cursor + # or, line 368 + try: + v_5 = self.limit - self.cursor + try: + # call mark_possessives, line 368 + if not self.r_mark_possessives(): + raise lab9() + raise lab8() + except lab9: pass + self.cursor = self.limit - v_5 + # call mark_sU, line 368 + if not self.r_mark_sU(): + raise lab7() + except lab8: pass + # ], line 368 + self.bra = self.cursor + # delete, line 368 + if not self.slice_del(): + return False + + # try, line 368 + v_6 = self.limit - self.cursor + try: + # (, line 368 + # [, line 368 + self.ket = self.cursor + # call mark_lAr, line 368 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_6 + raise lab10() + # ], line 368 + self.bra = self.cursor + # delete, line 368 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 368 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_6 + raise lab10() + except lab10: pass + raise lab5() + except lab7: pass + self.cursor = self.limit - v_4 + # (, line 370 + # [, line 370 + self.ket = self.cursor + # call mark_lAr, line 370 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_3 + raise lab4() + # ], line 370 + self.bra = self.cursor + # delete, line 370 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 370 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_3 + raise lab4() + except lab5: pass + except lab4: pass + raise lab0() + except lab3: pass + self.cursor = self.limit - v_1 + try: + # (, line 374 + # [, line 374 + self.ket = self.cursor + # (, line 374 + # or, line 374 + try: + v_7 = self.limit - self.cursor + try: + # call mark_ndA, line 374 + if not self.r_mark_ndA(): + raise lab13() + raise lab12() + except lab13: pass + self.cursor = self.limit - v_7 + # call mark_nA, line 374 + if not self.r_mark_nA(): + raise lab11() + except lab12: pass + # (, line 375 + # or, line 377 + try: + v_8 = self.limit - self.cursor + try: + # (, line 376 + # call mark_lArI, line 376 + if not self.r_mark_lArI(): + raise lab15() + # ], line 376 + self.bra = self.cursor + # delete, line 376 + if not self.slice_del(): + return False + + raise lab14() + except lab15: pass + self.cursor = self.limit - v_8 + try: + # (, line 378 + # call mark_sU, line 378 + if not self.r_mark_sU(): + raise lab16() + # ], line 378 + self.bra = self.cursor + # delete, line 378 + if not self.slice_del(): + return False + + # try, line 378 + v_9 = self.limit - self.cursor + try: + # (, line 378 + # [, line 378 + self.ket = self.cursor + # call mark_lAr, line 378 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_9 + raise lab17() + # ], line 378 + self.bra = self.cursor + # delete, line 378 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 378 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_9 + raise lab17() + except lab17: pass + raise lab14() + except lab16: pass + self.cursor = self.limit - v_8 + # (, line 380 + # call stem_suffix_chain_before_ki, line 380 + if not self.r_stem_suffix_chain_before_ki(): + raise lab11() + except lab14: pass + raise lab0() + except lab11: pass + self.cursor = self.limit - v_1 + try: + # (, line 384 + # [, line 384 + self.ket = self.cursor + # (, line 384 + # or, line 384 + try: + v_10 = self.limit - self.cursor + try: + # call mark_ndAn, line 384 + if not self.r_mark_ndAn(): + raise lab20() + raise lab19() + except lab20: pass + self.cursor = self.limit - v_10 + # call mark_nU, line 384 + if not self.r_mark_nU(): + raise lab18() + except lab19: pass + # (, line 384 + # or, line 384 + try: + v_11 = self.limit - self.cursor + try: + # (, line 384 + # call mark_sU, line 384 + if not self.r_mark_sU(): + raise lab22() + # ], line 384 + self.bra = self.cursor + # delete, line 384 + if not self.slice_del(): + return False + + # try, line 384 + v_12 = self.limit - self.cursor + try: + # (, line 384 + # [, line 384 + self.ket = self.cursor + # call mark_lAr, line 384 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_12 + raise lab23() + # ], line 384 + self.bra = self.cursor + # delete, line 384 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 384 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_12 + raise lab23() + except lab23: pass + raise lab21() + except lab22: pass + self.cursor = self.limit - v_11 + # (, line 384 + # call mark_lArI, line 384 + if not self.r_mark_lArI(): + raise lab18() + except lab21: pass + raise lab0() + except lab18: pass + self.cursor = self.limit - v_1 + try: + # (, line 386 + # [, line 386 + self.ket = self.cursor + # call mark_DAn, line 386 + if not self.r_mark_DAn(): + raise lab24() + # ], line 386 + self.bra = self.cursor + # delete, line 386 + if not self.slice_del(): + return False + + # try, line 386 + v_13 = self.limit - self.cursor + try: + # (, line 386 + # [, line 386 + self.ket = self.cursor + # (, line 387 + # or, line 389 + try: + v_14 = self.limit - self.cursor + try: + # (, line 388 + # call mark_possessives, line 388 + if not self.r_mark_possessives(): + raise lab27() + # ], line 388 + self.bra = self.cursor + # delete, line 388 + if not self.slice_del(): + return False + + # try, line 388 + v_15 = self.limit - self.cursor + try: + # (, line 388 + # [, line 388 + self.ket = self.cursor + # call mark_lAr, line 388 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_15 + raise lab28() + # ], line 388 + self.bra = self.cursor + # delete, line 388 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 388 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_15 + raise lab28() + except lab28: pass + raise lab26() + except lab27: pass + self.cursor = self.limit - v_14 + try: + # (, line 390 + # call mark_lAr, line 390 + if not self.r_mark_lAr(): + raise lab29() + # ], line 390 + self.bra = self.cursor + # delete, line 390 + if not self.slice_del(): + return False + + # try, line 390 + v_16 = self.limit - self.cursor + try: + # (, line 390 + # call stem_suffix_chain_before_ki, line 390 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_16 + raise lab30() + except lab30: pass + raise lab26() + except lab29: pass + self.cursor = self.limit - v_14 + # (, line 392 + # call stem_suffix_chain_before_ki, line 392 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_13 + raise lab25() + except lab26: pass + except lab25: pass + raise lab0() + except lab24: pass + self.cursor = self.limit - v_1 + try: + # (, line 396 + # [, line 396 + self.ket = self.cursor + # or, line 396 + try: + v_17 = self.limit - self.cursor + try: + # call mark_nUn, line 396 + if not self.r_mark_nUn(): + raise lab33() + raise lab32() + except lab33: pass + self.cursor = self.limit - v_17 + # call mark_ylA, line 396 + if not self.r_mark_ylA(): + raise lab31() + except lab32: pass + # ], line 396 + self.bra = self.cursor + # delete, line 396 + if not self.slice_del(): + return False + + # try, line 397 + v_18 = self.limit - self.cursor + try: + # (, line 397 + # or, line 399 + try: + v_19 = self.limit - self.cursor + try: + # (, line 398 + # [, line 398 + self.ket = self.cursor + # call mark_lAr, line 398 + if not self.r_mark_lAr(): + raise lab36() + # ], line 398 + self.bra = self.cursor + # delete, line 398 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 398 + if not self.r_stem_suffix_chain_before_ki(): + raise lab36() + raise lab35() + except lab36: pass + self.cursor = self.limit - v_19 + try: + # (, line 400 + # [, line 400 + self.ket = self.cursor + # or, line 400 + try: + v_20 = self.limit - self.cursor + try: + # call mark_possessives, line 400 + if not self.r_mark_possessives(): + raise lab39() + raise lab38() + except lab39: pass + self.cursor = self.limit - v_20 + # call mark_sU, line 400 + if not self.r_mark_sU(): + raise lab37() + except lab38: pass + # ], line 400 + self.bra = self.cursor + # delete, line 400 + if not self.slice_del(): + return False + + # try, line 400 + v_21 = self.limit - self.cursor + try: + # (, line 400 + # [, line 400 + self.ket = self.cursor + # call mark_lAr, line 400 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_21 + raise lab40() + # ], line 400 + self.bra = self.cursor + # delete, line 400 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 400 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_21 + raise lab40() + except lab40: pass + raise lab35() + except lab37: pass + self.cursor = self.limit - v_19 + # call stem_suffix_chain_before_ki, line 402 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_18 + raise lab34() + except lab35: pass + except lab34: pass + raise lab0() + except lab31: pass + self.cursor = self.limit - v_1 + try: + # (, line 406 + # [, line 406 + self.ket = self.cursor + # call mark_lArI, line 406 + if not self.r_mark_lArI(): + raise lab41() + # ], line 406 + self.bra = self.cursor + # delete, line 406 + if not self.slice_del(): + return False + + raise lab0() + except lab41: pass + self.cursor = self.limit - v_1 + try: + # (, line 408 + # call stem_suffix_chain_before_ki, line 408 + if not self.r_stem_suffix_chain_before_ki(): + raise lab42() + raise lab0() + except lab42: pass + self.cursor = self.limit - v_1 + try: + # (, line 410 + # [, line 410 + self.ket = self.cursor + # or, line 410 + try: + v_22 = self.limit - self.cursor + try: + # call mark_DA, line 410 + if not self.r_mark_DA(): + raise lab45() + raise lab44() + except lab45: pass + self.cursor = self.limit - v_22 + try: + # call mark_yU, line 410 + if not self.r_mark_yU(): + raise lab46() + raise lab44() + except lab46: pass + self.cursor = self.limit - v_22 + # call mark_yA, line 410 + if not self.r_mark_yA(): + raise lab43() + except lab44: pass + # ], line 410 + self.bra = self.cursor + # delete, line 410 + if not self.slice_del(): + return False + + # try, line 410 + v_23 = self.limit - self.cursor + try: + # (, line 410 + # [, line 410 + self.ket = self.cursor + # (, line 410 + # or, line 410 + try: + v_24 = self.limit - self.cursor + try: + # (, line 410 + # call mark_possessives, line 410 + if not self.r_mark_possessives(): + raise lab49() + # ], line 410 + self.bra = self.cursor + # delete, line 410 + if not self.slice_del(): + return False + + # try, line 410 + v_25 = self.limit - self.cursor + try: + # (, line 410 + # [, line 410 + self.ket = self.cursor + # call mark_lAr, line 410 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_25 + raise lab50() + except lab50: pass + raise lab48() + except lab49: pass + self.cursor = self.limit - v_24 + # call mark_lAr, line 410 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_23 + raise lab47() + except lab48: pass + # ], line 410 + self.bra = self.cursor + # delete, line 410 + if not self.slice_del(): + return False + + # [, line 410 + self.ket = self.cursor + # call stem_suffix_chain_before_ki, line 410 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_23 + raise lab47() + except lab47: pass + raise lab0() + except lab43: pass + self.cursor = self.limit - v_1 + # (, line 412 + # [, line 412 + self.ket = self.cursor + # or, line 412 + try: + v_26 = self.limit - self.cursor + try: + # call mark_possessives, line 412 + if not self.r_mark_possessives(): + raise lab52() + raise lab51() + except lab52: pass + self.cursor = self.limit - v_26 + # call mark_sU, line 412 + if not self.r_mark_sU(): + return False + except lab51: pass + # ], line 412 + self.bra = self.cursor + # delete, line 412 + if not self.slice_del(): + return False + + # try, line 412 + v_27 = self.limit - self.cursor + try: + # (, line 412 + # [, line 412 + self.ket = self.cursor + # call mark_lAr, line 412 + if not self.r_mark_lAr(): + self.cursor = self.limit - v_27 + raise lab53() + # ], line 412 + self.bra = self.cursor + # delete, line 412 + if not self.slice_del(): + return False + + # call stem_suffix_chain_before_ki, line 412 + if not self.r_stem_suffix_chain_before_ki(): + self.cursor = self.limit - v_27 + raise lab53() + except lab53: pass + except lab0: pass + return True + + def r_post_process_last_consonants(self): + # (, line 415 + # [, line 416 + self.ket = self.cursor + # substring, line 416 + among_var = self.find_among_b(TurkishStemmer.a_23, 4) + if among_var == 0: + return False + # ], line 416 + self.bra = self.cursor + if among_var == 0: + return False + elif among_var == 1: + # (, line 417 + # <-, line 417 + if not self.slice_from(u"p"): + return False + elif among_var == 2: + # (, line 418 + # <-, line 418 + if not self.slice_from(u"\u00E7"): + return False + elif among_var == 3: + # (, line 419 + # <-, line 419 + if not self.slice_from(u"t"): + return False + elif among_var == 4: + # (, line 420 + # <-, line 420 + if not self.slice_from(u"k"): + return False + return True + + def r_append_U_to_stems_ending_with_d_or_g(self): + # (, line 430 + # test, line 431 + v_1 = self.limit - self.cursor + # (, line 431 + # or, line 431 + try: + v_2 = self.limit - self.cursor + try: + # literal, line 431 + if not self.eq_s_b(1, u"d"): + raise lab1() + raise lab0() + except lab1: pass + self.cursor = self.limit - v_2 + # literal, line 431 + if not self.eq_s_b(1, u"g"): + return False + except lab0: pass + self.cursor = self.limit - v_1 + # or, line 433 + try: + v_3 = self.limit - self.cursor + try: + # (, line 432 + # test, line 432 + v_4 = self.limit - self.cursor + # (, line 432 + # (, line 432 + # goto, line 432 + try: + while True: + v_5 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab5() + self.cursor = self.limit - v_5 + raise lab4() + except lab5: pass + self.cursor = self.limit - v_5 + if self.cursor <= self.limit_backward: + raise lab3() + self.cursor -= 1 + except lab4: pass + # or, line 432 + try: + v_6 = self.limit - self.cursor + try: + # literal, line 432 + if not self.eq_s_b(1, u"a"): + raise lab7() + raise lab6() + except lab7: pass + self.cursor = self.limit - v_6 + # literal, line 432 + if not self.eq_s_b(1, u"\u0131"): + raise lab3() + except lab6: pass + self.cursor = self.limit - v_4 + # <+, line 432 + c = self.cursor + self.insert(self.cursor, self.cursor, u"\u0131") + self.cursor = c + raise lab2() + except lab3: pass + self.cursor = self.limit - v_3 + try: + # (, line 434 + # test, line 434 + v_7 = self.limit - self.cursor + # (, line 434 + # (, line 434 + # goto, line 434 + try: + while True: + v_8 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab10() + self.cursor = self.limit - v_8 + raise lab9() + except lab10: pass + self.cursor = self.limit - v_8 + if self.cursor <= self.limit_backward: + raise lab8() + self.cursor -= 1 + except lab9: pass + # or, line 434 + try: + v_9 = self.limit - self.cursor + try: + # literal, line 434 + if not self.eq_s_b(1, u"e"): + raise lab12() + raise lab11() + except lab12: pass + self.cursor = self.limit - v_9 + # literal, line 434 + if not self.eq_s_b(1, u"i"): + raise lab8() + except lab11: pass + self.cursor = self.limit - v_7 + # <+, line 434 + c = self.cursor + self.insert(self.cursor, self.cursor, u"i") + self.cursor = c + raise lab2() + except lab8: pass + self.cursor = self.limit - v_3 + try: + # (, line 436 + # test, line 436 + v_10 = self.limit - self.cursor + # (, line 436 + # (, line 436 + # goto, line 436 + try: + while True: + v_11 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab15() + self.cursor = self.limit - v_11 + raise lab14() + except lab15: pass + self.cursor = self.limit - v_11 + if self.cursor <= self.limit_backward: + raise lab13() + self.cursor -= 1 + except lab14: pass + # or, line 436 + try: + v_12 = self.limit - self.cursor + try: + # literal, line 436 + if not self.eq_s_b(1, u"o"): + raise lab17() + raise lab16() + except lab17: pass + self.cursor = self.limit - v_12 + # literal, line 436 + if not self.eq_s_b(1, u"u"): + raise lab13() + except lab16: pass + self.cursor = self.limit - v_10 + # <+, line 436 + c = self.cursor + self.insert(self.cursor, self.cursor, u"u") + self.cursor = c + raise lab2() + except lab13: pass + self.cursor = self.limit - v_3 + # (, line 438 + # test, line 438 + v_13 = self.limit - self.cursor + # (, line 438 + # (, line 438 + # goto, line 438 + try: + while True: + v_14 = self.limit - self.cursor + try: + if not self.in_grouping_b(TurkishStemmer.g_vowel, 97, 305): + raise lab19() + self.cursor = self.limit - v_14 + raise lab18() + except lab19: pass + self.cursor = self.limit - v_14 + if self.cursor <= self.limit_backward: + return False + self.cursor -= 1 + except lab18: pass + # or, line 438 + try: + v_15 = self.limit - self.cursor + try: + # literal, line 438 + if not self.eq_s_b(1, u"\u00F6"): + raise lab21() + raise lab20() + except lab21: pass + self.cursor = self.limit - v_15 + # literal, line 438 + if not self.eq_s_b(1, u"\u00FC"): + return False + except lab20: pass + self.cursor = self.limit - v_13 + # <+, line 438 + c = self.cursor + self.insert(self.cursor, self.cursor, u"\u00FC") + self.cursor = c + except lab2: pass + return True + + def r_more_than_one_syllable_word(self): + # (, line 445 + # test, line 446 + v_1 = self.cursor + # (, line 446 + # atleast, line 446 + v_2 = 2 + # atleast, line 446 + try: + while True: + try: + v_3 = self.cursor + try: + # (, line 446 + # gopast, line 446 + try: + while True: + try: + if not self.in_grouping(TurkishStemmer.g_vowel, 97, 305): + raise lab4() + raise lab3() + except lab4: pass + if self.cursor >= self.limit: + raise lab2() + self.cursor += 1 + except lab3: pass + v_2 -= 1 + raise lab1() + except lab2: pass + self.cursor = v_3 + raise lab0() + except lab1: pass + except lab0: pass + if v_2 > 0: + return False + self.cursor = v_1 + return True + + def r_is_reserved_word(self): + # (, line 449 + # or, line 451 + try: + v_1 = self.cursor + try: + # test, line 450 + v_2 = self.cursor + # (, line 450 + # gopast, line 450 + try: + while True: + try: + # literal, line 450 + if not self.eq_s(2, u"ad"): + raise lab3() + raise lab2() + except lab3: pass + if self.cursor >= self.limit: + raise lab1() + self.cursor += 1 + except lab2: pass + # (, line 450 + self.I_strlen = 2; + # (, line 450 + if not self.I_strlen == self.limit: + raise lab1() + self.cursor = v_2 + raise lab0() + except lab1: pass + self.cursor = v_1 + # test, line 452 + v_4 = self.cursor + # (, line 452 + # gopast, line 452 + try: + while True: + try: + # literal, line 452 + if not self.eq_s(5, u"soyad"): + raise lab5() + raise lab4() + except lab5: pass + if self.cursor >= self.limit: + return False + self.cursor += 1 + except lab4: pass + # (, line 452 + self.I_strlen = 5; + # (, line 452 + if not self.I_strlen == self.limit: + return False + self.cursor = v_4 + except lab0: pass + return True + + def r_postlude(self): + # (, line 455 + # not, line 456 + v_1 = self.cursor + try: + # (, line 456 + # call is_reserved_word, line 456 + if not self.r_is_reserved_word(): + raise lab0() + return False + except lab0: pass + self.cursor = v_1 + # backwards, line 457 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 457 + # do, line 458 + v_2 = self.limit - self.cursor + try: + # call append_U_to_stems_ending_with_d_or_g, line 458 + if not self.r_append_U_to_stems_ending_with_d_or_g(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + # do, line 459 + v_3 = self.limit - self.cursor + try: + # call post_process_last_consonants, line 459 + if not self.r_post_process_last_consonants(): + raise lab2() + except lab2: pass + self.cursor = self.limit - v_3 + self.cursor = self.limit_backward + return True + + def _stem(self): + # (, line 464 + # (, line 465 + # call more_than_one_syllable_word, line 465 + if not self.r_more_than_one_syllable_word(): + return False + # (, line 466 + # backwards, line 467 + self.limit_backward = self.cursor + self.cursor = self.limit + # (, line 467 + # do, line 468 + v_1 = self.limit - self.cursor + try: + # call stem_nominal_verb_suffixes, line 468 + if not self.r_stem_nominal_verb_suffixes(): + raise lab0() + except lab0: pass + self.cursor = self.limit - v_1 + # Boolean test continue_stemming_noun_suffixes, line 469 + if not self.B_continue_stemming_noun_suffixes: + return False + # do, line 470 + v_2 = self.limit - self.cursor + try: + # call stem_noun_suffixes, line 470 + if not self.r_stem_noun_suffixes(): + raise lab1() + except lab1: pass + self.cursor = self.limit - v_2 + self.cursor = self.limit_backward + # call postlude, line 473 + if not self.r_postlude(): + return False + return True + + def equals(self, o): + return isinstance(o, TurkishStemmer) + + def hashCode(self): + return hash("TurkishStemmer") +class lab0(BaseException): pass +class lab1(BaseException): pass +class lab2(BaseException): pass +class lab3(BaseException): pass +class lab4(BaseException): pass +class lab5(BaseException): pass +class lab6(BaseException): pass +class lab7(BaseException): pass +class lab8(BaseException): pass +class lab9(BaseException): pass +class lab10(BaseException): pass +class lab11(BaseException): pass +class lab12(BaseException): pass +class lab13(BaseException): pass +class lab14(BaseException): pass +class lab15(BaseException): pass +class lab16(BaseException): pass +class lab17(BaseException): pass +class lab18(BaseException): pass +class lab19(BaseException): pass +class lab20(BaseException): pass +class lab21(BaseException): pass +class lab22(BaseException): pass +class lab23(BaseException): pass +class lab24(BaseException): pass +class lab25(BaseException): pass +class lab26(BaseException): pass +class lab27(BaseException): pass +class lab28(BaseException): pass +class lab29(BaseException): pass +class lab30(BaseException): pass +class lab31(BaseException): pass +class lab32(BaseException): pass +class lab33(BaseException): pass +class lab34(BaseException): pass +class lab35(BaseException): pass +class lab36(BaseException): pass +class lab37(BaseException): pass +class lab38(BaseException): pass +class lab39(BaseException): pass +class lab40(BaseException): pass +class lab41(BaseException): pass +class lab42(BaseException): pass +class lab43(BaseException): pass +class lab44(BaseException): pass +class lab45(BaseException): pass +class lab46(BaseException): pass +class lab47(BaseException): pass +class lab48(BaseException): pass +class lab49(BaseException): pass +class lab50(BaseException): pass +class lab51(BaseException): pass +class lab52(BaseException): pass +class lab53(BaseException): pass diff --git a/lint.py b/lint.py index 272a9d9..db188e5 100644 --- a/lint.py +++ b/lint.py @@ -17,24 +17,22 @@ pass # Add 'contrib' to sys.path to simulate installation of package 'flake8' -# and it's dependencies: 'pyflake', 'pep8', 'mccabe' and 'pep8-naming' +# and it's dependencies: 'pyflake', 'pycodestyle', 'mccabe' and 'pep8-naming' CONTRIB_PATH = os.path.join(os.path.dirname(__file__), 'contrib') if CONTRIB_PATH not in sys.path: sys.path.insert(0, CONTRIB_PATH) from flake8 import __version__ as flake8_version -from flake8._pyflakes import patch_pyflakes +from flake8.plugins.pyflakes import patch_pyflakes import flake8_debugger -from flake8_import_order import ( - __version__ as flake8_import_order_version, - ImportOrderChecker -) +from flake8_import_order import __version__ as flake8_import_order_version +from flake8_import_order.checker import ImportOrderChecker import mccabe -import pep8 +import pycodestyle import pep8ext_naming from pydocstyle import ( __version__ as pydocstyle_version, - PEP257Checker + ConventionChecker as PEP257Checker ) from pyflakes import ( __version__ as pyflakes_version, @@ -57,7 +55,7 @@ def tools_versions(): """Return all lint tools versions.""" return ( - ('pep8', pep8.__version__), + ('pycodestyle', pycodestyle.__version__), ('flake8', flake8_version), ('pyflakes', pyflakes_version), ('mccabe', mccabe.__version__), @@ -68,7 +66,7 @@ def tools_versions(): ) -class Pep8Report(pep8.BaseReport): +class Pep8Report(pycodestyle.BaseReport): """Collect all check results.""" def __init__(self, options): @@ -194,7 +192,7 @@ def lint(lines, settings): # lint with pep8 if settings.get('pep8', True): - pep8style = pep8.StyleGuide( + pep8style = pycodestyle.StyleGuide( reporter=Pep8Report, ignore=['DIRTY-HACK'], # PEP8 error will never starts like this max_line_length=settings.get('pep8_max_line_length') @@ -375,7 +373,7 @@ def lint_external(lines, settings, interpreter, linter): arg_parser.add_argument('--builtins', help="python builtins extend") arg_parser.add_argument('--pep8', action='store_true', - help="run pep8 lint") + help="run pycodestyle (pep8) lint") arg_parser.add_argument('--pydocstyle', action='store_true', help="run pydocstyle lint") arg_parser.add_argument('--naming', action='store_true',