diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4376256..17563af 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -9,7 +9,7 @@ jobs:
runners: '[
"ubuntu-24.04"
]'
- min-version: "3.8"
+ min-version: "3.9"
include-pre-releases: true
implementations: '["CPython", "PyPy"]'
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f9905a2..c02d0b2 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,20 +1,15 @@
repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
+ - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- - id: trailing-whitespace
- - id: end-of-file-fixer
- - id: check-yaml
- - id: check-added-large-files
+ - id: trailing-whitespace
+ - id: end-of-file-fixer
+ - id: check-yaml
+ - id: check-added-large-files
-- repo: https://github.com/psf/black
- rev: 25.1.0
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.16.2
hooks:
- - id: black
- args: ["-l", "79"]
-
-- repo: https://github.com/pycqa/flake8
- rev: 7.3.0
- hooks:
- - id: flake8
- args: ["--ignore", "E203,W503"]
+ - id: ruff-check
+ args: [ --fix ]
+ - id: ruff-format
diff --git a/pyproject.toml b/pyproject.toml
index 5929306..d5d121d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,18 +4,21 @@ build-backend = "hatchling.build"
[project]
name = "yabi-bython"
-version = "0.9.3"
+dynamic = ["version"]
description = "Yet Another Bython (braced Python) Implementation"
authors = [{name = "Sóla Łuset"}]
readme = "README.md"
license = "MIT"
license-files = ["LICENSE.md"]
-dependencies = ["pwcp>=0.12.2,<0.13"]
+dependencies = ["pwcp~=0.13.0"]
optional-dependencies = {tests = ["pytest"]}
-scripts = {yabi = "yabi:main", yabi-convert = "yabi:convert_main"}
+scripts = {yabi = "yabi:main", yabi-convert = "yabi.convert:main"}
[project.urls]
Repository = "https://github.com/solaluset/yabi"
[tool.hatch.build.targets.wheel]
packages = ["yabi"]
+
+[tool.hatch.version]
+path = "yabi/version.py"
diff --git a/ruff.toml b/ruff.toml
new file mode 100644
index 0000000..28d6185
--- /dev/null
+++ b/ruff.toml
@@ -0,0 +1 @@
+line-length = 79
diff --git a/tests/test_yabi.py b/tests/test_yabi.py
index dcd915c..84933d2 100644
--- a/tests/test_yabi.py
+++ b/tests/test_yabi.py
@@ -1,21 +1,51 @@
+import ast
import os
import sys
from io import StringIO
from unittest.mock import patch
-from pytest import mark
+from pytest import fixture, mark
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
-from yabi import config, main, to_bython, to_pure_python # noqa: E402
-from yabi.console import YabiConsole # noqa: E402
-
+from yabi import config, main, to_bython, to_pure_python
+from yabi.console import YabiConsole
+from yabi.parser import _transform
sys.dont_write_bytecode = True
sys.ps1 = getattr(sys, "ps1", ">>> ")
sys.ps2 = getattr(sys, "ps2", "... ")
+def _ok_err(func, *args):
+ try:
+ return func(*args), None
+ except Exception as e: # noqa: BLE001
+ return None, e
+
+
+@fixture(scope="session", autouse=True)
+def checked_transform():
+ def _transform_test(code: str, python: bool) -> str:
+ ret = _transform(code, python)
+
+ result1 = _transform(ret, True)
+ result2 = ret if python else _transform(result1, True)
+
+ ast1, err1 = _ok_err(ast.parse, result1)
+ ast2, err2 = _ok_err(ast.parse, result2)
+ # check if ASTs or their errors are equivalent
+ if ast1:
+ assert ast2 and ast.dump(ast1) == ast.dump(ast2)
+ else:
+ assert type(err1) is type(err2) and err1.args == err2.args
+
+ return ret
+
+ with patch("yabi.parser._transform", new=_transform_test):
+ yield
+
+
def check_file(file, expexted_output, *args):
with patch("sys.stdout", new=StringIO()):
main(args + (file,))
@@ -23,8 +53,9 @@ def check_file(file, expexted_output, *args):
def check_console(code, expexted_output):
- with patch("sys.stdout", new=StringIO()), patch(
- "sys.stdin", new=StringIO(code)
+ with (
+ patch("sys.stdout", new=StringIO()),
+ patch("sys.stdin", new=StringIO(code)),
):
YabiConsole().interact()
assert sys.stdout.getvalue() == expexted_output
@@ -126,6 +157,19 @@ def test_console_linecont():
check_console(code, expexted_output)
+def test_console_preprocessing():
+ code = (
+ """
+#pragma pypp on
+#define a 1
+a
+ """.strip()
+ + "\n"
+ )
+ expexted_output = ">>> >>> >>> 1\n>>> "
+ check_console(code, expexted_output)
+
+
def test_to_bython():
assert (
to_bython(
@@ -138,9 +182,7 @@ def test_to_bython():
!
/* these comments too */
print("No")
- """.strip().replace(
- "!", ""
- )
+ """.strip().replace("!", "")
)
== """
for (i in {1, 2, 3}) {
@@ -154,10 +196,7 @@ def test_to_bython():
print("No")
}
}
- """.strip().replace(
- "!", ""
- )
- + "\n"
+ """.strip().replace("!", "")
)
@@ -166,7 +205,17 @@ def test_comment_with_semicolon():
def test_semicolon_parse():
- assert to_pure_python("; while True {}") == "while True:\n pass\n"
+ assert to_pure_python("; while True {}\n") == "while True:\n pass\n"
+ assert (
+ to_pure_python("if True: if True: pass; while True: pass")
+ == """
+if True:
+ if True:
+ pass
+ while True:
+ pass
+ """.strip()
+ )
def test_inline_comment():
@@ -177,6 +226,25 @@ def test_inline_comment():
assert to_pure_python(code).strip() == code
+def test_c_comment():
+ assert to_pure_python("/* test */") == "# test"
+ assert (
+ to_pure_python(
+ """
+/* one
+ more
+ test */
+ """
+ )
+ == """
+# one
+# more
+# test
+ """.strip()
+ + "\n"
+ )
+
+
def test_preprocessing():
with open("tests/preprocess.by") as f:
code = f.read()
diff --git a/yabi/__init__.py b/yabi/__init__.py
index 2d4298b..a90f896 100644
--- a/yabi/__init__.py
+++ b/yabi/__init__.py
@@ -1,12 +1,10 @@
__all__ = (
"__version__",
"main",
- "convert_main",
"to_bython",
"to_pure_python",
)
-from .runner import __version__, main
-from .convert import convert_main
from .parser import to_bython, to_pure_python
+from .runner import __version__, main
diff --git a/yabi/__main__.py b/yabi/__main__.py
index e7d3e76..8273c4f 100644
--- a/yabi/__main__.py
+++ b/yabi/__main__.py
@@ -1,4 +1,3 @@
from . import main
-
main()
diff --git a/yabi/console.py b/yabi/console.py
index 51d31a4..ac6eb96 100644
--- a/yabi/console.py
+++ b/yabi/console.py
@@ -1,34 +1,26 @@
-import sys
import ast
+import sys
from code import InteractiveConsole
-from pwcp.preprocessor import PyPreprocessor, PreprocessorError, preprocess
+from pwcp.preprocessor import PreprocessorError, preprocess
-from . import config
-from .parser import UNCLOSED_BLOCK_ERROR, to_pure_python
+from .parser import UNCLOSED_BLOCK_ERROR
class YabiConsole(InteractiveConsole):
def __init__(self):
super().__init__()
- self.preprocessor = PyPreprocessor(
- disabled=not config.ENABLE_PREPROCESSING
- )
+ self.pwcp_data = {}
def runsource(self, source, filename="", symbol="single") -> bool:
try:
- parsed = to_pure_python(
- preprocess(source, filename, self.preprocessor)[0]
- )
- except PreprocessorError:
- self.showtraceback()
- return False
- except SyntaxError as e:
+ parsed = preprocess(source, filename, self.pwcp_data)[0]
+ except (PreprocessorError, SyntaxError) as e:
if e.args[0] == UNCLOSED_BLOCK_ERROR or e.args[0].startswith(
"Unterminated"
):
return True
- self.showtraceback()
+ self.showsyntaxerror(filename)
return False
if not source.endswith("\n"):
parsed = parsed.rstrip("\n")
@@ -56,11 +48,11 @@ def runsource(self, source, filename="", symbol="single") -> bool:
def runcode(self, code) -> bool:
try:
- exec(code, self.locals)
+ exec(code, self.locals) # noqa: S102
return True
except SystemExit:
raise
- except BaseException:
+ except BaseException: # noqa: BLE001
self.showtraceback()
return False
@@ -69,9 +61,6 @@ def _compiler(self, source, filename, symbol, original_source):
return None
try:
return self.compile(source, filename, symbol)
- except PreprocessorError:
- self.showtraceback()
- return self.compile("", filename, symbol)
except SyntaxError:
if "\n" in original_source and not source.endswith("\n"):
return None
diff --git a/yabi/convert.py b/yabi/convert.py
index 4fd21b0..6180b53 100644
--- a/yabi/convert.py
+++ b/yabi/convert.py
@@ -1,8 +1,7 @@
-import sys
import argparse
+import sys
-from .parser import to_pure_python, to_bython
-
+from .parser import to_bython, to_pure_python
parser = argparse.ArgumentParser(
description="Python <==> Bython converter",
@@ -16,8 +15,12 @@
parser.add_argument("target")
-def convert_main(args=sys.argv[1:]):
+def main(args=sys.argv[1:]):
args = parser.parse_args(args)
converter = to_pure_python if args.to_python else to_bython
with open(args.target) as source:
print(converter(source.read()).rstrip("\n"))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/yabi/hooks.py b/yabi/hooks.py
new file mode 100644
index 0000000..daa67ee
--- /dev/null
+++ b/yabi/hooks.py
@@ -0,0 +1,27 @@
+from pwcp import PreprocessorHooks, PycType
+
+from . import config
+from .parser import to_pure_python
+from .version import __version__
+
+
+class YabiHooks(PreprocessorHooks):
+ def __init__(self):
+ super().__init__("yabi")
+
+ def process_source(
+ self, source: str, filename: str, state: None
+ ) -> tuple[str, None]:
+ return to_pure_python(source), None
+
+ def create_pyc_data(self, data: None, pyc_type: PycType) -> dict:
+ return {
+ "version": __version__,
+ "preprocessing": config.ENABLE_PREPROCESSING,
+ }
+
+ def validate_pyc_data(self, pyc: dict, pyc_type: PycType) -> bool:
+ return (
+ pyc["version"] == __version__
+ and pyc["preprocessing"] == config.ENABLE_PREPROCESSING
+ )
diff --git a/yabi/parser.py b/yabi/parser.py
index bb9634d..8166c37 100644
--- a/yabi/parser.py
+++ b/yabi/parser.py
@@ -1,15 +1,12 @@
from __future__ import annotations
+
import ast
-import random
+from collections.abc import Generator, Iterable
from io import StringIO
-from builtins import compile
-from string import ascii_letters
-from typing import Generator, Iterable
from tokenize import NAME, OP, generate_tokens
from pypp.parser import default_lexer
-
KEYWORDS = {
"async",
"class",
@@ -37,6 +34,10 @@
UNCLOSED_BLOCK_ERROR = "there is an unclosed block"
+class YabiSyntaxError(SyntaxError):
+ pass
+
+
def tokenize(text: str) -> Generator[str, None, None]:
lexer = default_lexer()
lexer.input(text)
@@ -68,6 +69,13 @@ def strip_spaces(tokens: list[str]) -> None:
del tokens[-1]
+def to_python_comment(part: str) -> str:
+ if not part.startswith("/*") or not part.endswith("*/"):
+ return part
+ part = part.replace("/*", "#", 1).removesuffix("*/").rstrip()
+ return part.replace("\n", "\n#")
+
+
class Block:
def __init__(self):
self.head = []
@@ -118,7 +126,7 @@ def finish(self):
self.body[-1].finish()
else:
if self.finished:
- raise SyntaxError("the block was already closed")
+ raise YabiSyntaxError("the block was already closed")
self.finished = True
def reindent(self, indent: str):
@@ -135,7 +143,7 @@ def reindent(self, indent: str):
):
del self.body[i - 1]
i -= 1
- if i + 1 == len(self.body) or self.body[i + 1] != "\n":
+ if i + 1 < len(self.body) and self.body[i + 1] != "\n":
self.body.insert(i + 1, "\n")
elif self.body[i] == "\n":
after_nl = True
@@ -183,7 +191,7 @@ def unparse(self, pure_python=True, depth=0) -> str:
(
child.unparse(pure_python, depth + 1)
if isinstance(child, Block)
- else child
+ else (to_python_comment(child) if pure_python else child)
)
for child in self.body
)
@@ -279,8 +287,7 @@ def _insert_into_line(line: str, index: int, part: str):
def _add_return(code: str) -> str:
- # same as `ast.parse`, but with non-overriden compile
- tree = compile(code, "", "exec", ast.PyCF_ONLY_AST)
+ tree = ast.parse(code)
last_node = tree.body[0].body[-1]
if not isinstance(last_node, ast.Expr):
return code
@@ -298,8 +305,14 @@ def _add_return(code: str) -> str:
return "\n".join(code)
+_lambda_count = 0
+
+
def _gen_lambda_name() -> str:
- return "_yabi_lambda_" + "".join(random.choices(ascii_letters, k=16))
+ global _lambda_count
+
+ _lambda_count += 1
+ return f"_yabi_lambda_{_lambda_count:016x}"
class Parser:
@@ -309,7 +322,7 @@ def __init__(self, tokens: Iterable[str]):
self.result = None
self.in_head = False
self.after_colon = False
- self.finish_on_nl = False
+ self.finish_on_nl = 0
self.capture_indent = False
self.after_indent = False
self.after_nl = True
@@ -318,7 +331,6 @@ def __init__(self, tokens: Iterable[str]):
self.seen_lambdas = 0
self.block_started = False
self.skip = False
- self.next_indent = None
self.head_term = None
self.tokens = [tok for tok in tokens if tok]
self.i = 0
@@ -333,11 +345,6 @@ def parse(self):
def _parse(self):
while self.i < len(self.tokens):
tok = self.tokens[self.i]
- if self.next_indent is not None:
- if not tok.isspace():
- self.i -= 1
- tok = self.next_indent
- self.next_indent = None
self.block_started = False
if tok == "#":
while (
@@ -347,19 +354,20 @@ def _parse(self):
self.i += 1
continue
if self.after_colon:
+ self.accept_keyword = True
self._parse_after_colon(tok)
elif self.after_indent:
self._parse_after_indent(tok)
elif self.after_nl:
self._parse_after_nl(tok)
- if tok == ";" and not self.finish_on_nl:
+ if tok == ";":
tok = "\n"
- self.next_indent = self.indent_stack[-1] or None
- if tok == "\n":
self.after_nl = True
- if self.finish_on_nl:
- self.finish_on_nl = False
+ elif tok == "\n":
+ self.after_nl = True
+ for _ in range(self.finish_on_nl):
self.result.finish()
+ self.finish_on_nl = 0
elif tok == "async" and self._next_nonspace(self.i) == "lambda":
self.async_lambda = True
self.i += 1
@@ -382,12 +390,10 @@ def _parse(self):
self.result.append(tok)
self.i += 1
- for indent in self.indent_stack:
- if indent is None:
- raise SyntaxError(UNCLOSED_BLOCK_ERROR)
+ if any(indent is None for indent in self.indent_stack):
+ raise YabiSyntaxError(UNCLOSED_BLOCK_ERROR)
+ while not self.result.finished:
self.result.finish()
- if not self.result.finished:
- raise SyntaxError(UNCLOSED_BLOCK_ERROR)
def _next_nonspace(self, i: int) -> str | None:
return next(
@@ -405,7 +411,7 @@ def _parse_after_colon(self, tok: str):
self.capture_indent = True
elif not tok.isspace():
self.after_colon = False
- self.finish_on_nl = True
+ self.finish_on_nl += 1
def _parse_after_indent(self, tok: str):
self.after_indent = False
@@ -458,7 +464,7 @@ def _parse_long_lambda(self, tok: str) -> bool:
self.i += 1
return True
if terminator != "{":
- raise SyntaxError("async lambda must use braces")
+ raise YabiSyntaxError("async lambda must use braces")
lambda_body = self._fully_parse_long_lambda()
brace_stack_copy = []
for brace, is_block in reversed(self.brace_stack):
@@ -493,7 +499,7 @@ def _fully_parse_long_lambda(self) -> Block:
head.append(tok)
self.i += 1
if brace_stack:
- raise SyntaxError(UNCLOSED_BLOCK_ERROR)
+ raise YabiSyntaxError(UNCLOSED_BLOCK_ERROR)
name = _gen_lambda_name()
if self.in_head:
@@ -502,7 +508,7 @@ def _fully_parse_long_lambda(self) -> Block:
self.result.append(name)
strip_spaces(head)
- if not head or not head[0] == "(" or not head[-1] == ")":
+ if not head or head[0] != "(" or head[-1] != ")":
head.insert(0, "(")
head.append(")")
head = ["def", " ", name] + head
@@ -545,21 +551,21 @@ def _parse_tok_in_braces_values(self, tok: str):
try:
brace, block_finished = self.brace_stack.pop()
except IndexError as e:
- raise SyntaxError(f"unmatched '{tok}'") from e
+ raise YabiSyntaxError(f"unmatched '{tok}'") from e
self.accept_keyword = block_finished
if BRACES[brace] != tok:
- raise SyntaxError(
+ raise YabiSyntaxError(
f"closing parenthesis '{tok}' does not match"
f" opening parenthesis '{brace}'"
)
if block_finished:
- if self.finish_on_nl:
- self.finish_on_nl = False
+ for _ in range(self.finish_on_nl):
self.result.finish()
+ self.finish_on_nl = 0
self.result.finish()
self.skip = True
if self.indent_stack.pop() is not None:
- raise SyntaxError("indented block was not properly closed")
+ raise YabiSyntaxError("indented block was not properly closed")
def parse(tokens: Iterable[str]) -> Block:
@@ -568,7 +574,7 @@ def parse(tokens: Iterable[str]) -> Block:
def _transform(code: str, python: bool) -> str:
- result = parse(tokenize(code + "\n"))
+ result = parse(tokenize(code))
return result.unparse(python)
diff --git a/yabi/runner.py b/yabi/runner.py
index ecf11e8..b835d7d 100644
--- a/yabi/runner.py
+++ b/yabi/runner.py
@@ -1,15 +1,12 @@
+import argparse
import os
import sys
-import argparse
-from importlib import metadata
import pwcp
from . import config
-from .parser import to_pure_python
-
-
-__version__ = metadata.version("yabi-bython")
+from .hooks import YabiHooks
+from .version import __version__
parser = argparse.ArgumentParser(
(
@@ -27,7 +24,7 @@
"-c", action="store_true", help="run target as command line"
)
parser.add_argument(
- "--prefer-py",
+ "--prefer-python",
dest="prefer_python",
action="store_true",
help="prefer .py files over .by when importing",
@@ -51,25 +48,19 @@
def main(args=sys.argv[1:]):
args = parser.parse_args(args)
- pwcp.add_file_extension(config.EXTENSION)
-
- def preprocess(src, filename, preprocessor):
- if not config.ENABLE_PREPROCESSING and filename.endswith(
- config.EXTENSION
- ):
- preprocessor.disabled = True
- return to_pure_python(orig_preprocess(src, filename, preprocessor))
-
- orig_preprocess = pwcp.set_preprocessing_function(preprocess)
-
config.SAVE_FILES = args.save_files
config.ENABLE_PREPROCESSING = args.enable_preprocessing
del args.enable_preprocessing
+ pwcp.add_file_extension(
+ config.EXTENSION, preprocess=config.ENABLE_PREPROCESSING
+ )
+ pwcp.add_hook(YabiHooks())
+
if not args.target:
args.m = True
args.target = f"{__package__}.console"
pwcp.main_with_params(
- **vars(args), preprocess_unknown_sources=config.ENABLE_PREPROCESSING
+ **vars(args), skip_unknown_sources=not config.ENABLE_PREPROCESSING
)
diff --git a/yabi/version.py b/yabi/version.py
new file mode 100644
index 0000000..61fb31c
--- /dev/null
+++ b/yabi/version.py
@@ -0,0 +1 @@
+__version__ = "0.10.0"