feat: add guarded C-family compaction

examples/Sample.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+
+#nullable enable
+namespace Demo
+{
+    public class Sample
+    {
+        private readonly Dictionary<string, List<int>> _index = new();
+        private const string Path = @"C:\dir\file  with  spaces.txt";
+
+        public static void Main(string[] args)
+        {
+            int i = 10;
+            var name = "world";
+            string msg = $"hello {name}, i = {i}";
+            string? maybe = null;
+            var len = maybe?.Length ?? 0;
+            int @class = 3;
+            var range = new[] { 1, 2, 3, 4 }[1..^1];
+            Func<int, int> inc = x => x + 1;
+            int neg = - -i;
+            if (i is not 0 && len >= 0)
+            {
+                Console.WriteLine(msg + @"verbatim ""with quotes""");
+            }
+            var raw = """
+                raw   text
+                """;
+        }
+    }
+}

examples/Sample.java

@@ -0,0 +1,36 @@
+package demo;
+
+import java.util.*;
+import java.util.function.Function;
+
+
+@SuppressWarnings("unchecked")
+public class Sample<T extends Comparable<T>> {
+    private final Map<String, List<Integer>> index = new HashMap<>();
+    private static final String BLOCK = """
+        line 1
+          line 2 with  spaces
+        """;
+
+    public static void main(String[] args) {
+        int i = 10;
+        while (i-- > 0) {
+            if (i % 2 == 0 && i != 4) continue;
+            System.out.println("i = " + i + " 'aspas' \"escaped\"");
+        }
+        Function<Integer, Integer> inc = x -> x + 1;
+        int neg = - -i;
+        long big = 1_000_000L;
+        double d = 1e-3 + .5;
+        char c = '\'';
+        List<List<String>> nested = new ArrayList<>();
+        Runnable r = Sample::helper;
+        r.run();
+    }
+
+    static void helper() {
+        int a = 1, b = 2;
+        a = a + +b;
+        b = a - --b;
+    }
+}

examples/sample.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(void) {
+    int value = 2 + 3;
+    printf("%d\n", value);
+    return 0;
+}

examples/sample.cpp

@@ -0,0 +1,42 @@
+#include <cstdio>
+#include <vector>
+#include <string>
+
+#define SQUARE(x) \
+    ((x) * (x))
+
+using namespace std;
+
+template <typename T>
+struct Box {
+    T value;
+    Box(T v) : value(v) {}
+    T get() const { return value; }
+};
+
+static const char* RAW = R"txt(
+   raw   text with "quotes" and \ backslashes
+)txt";
+
+int compute(int a, int b) {
+    int r = a - -b;
+    r = r + +a;
+    r -= b;
+    vector<vector<int> > grid(3, vector<int>(3, 0));
+    vector<vector<int>> grid2(2, vector<int>(2, 1));
+    for (int i = 0; i < 3; ++i)
+        for (int j = 0; j < 3; ++j)
+            grid[i][j] = i * j;
+    int big = 1'000'000;
+    double d = 1.5e+3;
+    unsigned x = 0xE + 1;
+    return r + grid[1][2] + big + (int)d + (int)x + SQUARE(3);
+}
+
+int main() {
+    Box<int> b(compute(3, 4));
+    printf("value = %d  (%s)\n", b.get(), "text   with   spaces");
+    int k = 10;
+    while (k --> 0) { if (k % 3) continue; }
+    return 0;
+}

llm_compact/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

llm_compact/cli.py

@@ -0,0 +1,68 @@
+import argparse
+import sys
+from pathlib import Path
+
+from .core import SUPPORTED_LANGUAGES, CompactOptions, compact
+from .tokens import DEFAULT_ENCODINGS, compare_tokens
+
+EXTENSIONS = {
+    ".py": "python",
+    ".java": "java",
+    ".c": "c",
+    ".h": "c",
+    ".cpp": "cpp",
+    ".cc": "cpp",
+    ".cxx": "cpp",
+    ".hpp": "cpp",
+    ".cs": "csharp",
+}
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(
+        prog="llm-compact",
+        description="Compact source code before sending it to an LLM and report the token saving.",
+    )
+    ap.add_argument("file", help="source file, or - to read standard input")
+    ap.add_argument("-l", "--language", choices=SUPPORTED_LANGUAGES, help="default: from extension")
+    ap.add_argument("--level", choices=("full", "light"), default="full")
+    ap.add_argument("--strip-comments", action="store_true")
+    ap.add_argument("-o", "--output", help="write the result here instead of standard output")
+    ap.add_argument("--no-stats", action="store_true", help="do not print the token report")
+    return ap
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.file == "-":
+        source = sys.stdin.read()
+        lang = args.language
+    else:
+        path = Path(args.file)
+        source = path.read_text(encoding="utf-8")
+        lang = args.language or EXTENSIONS.get(path.suffix.lower())
+    if lang is None:
+        print("error: could not infer the language, use --language", file=sys.stderr)
+        return 2
+
+    result = compact(source, lang, CompactOptions(args.level, args.strip_comments))
+    if args.output:
+        Path(args.output).write_text(result.code, encoding="utf-8")
+    else:
+        sys.stdout.write(result.code)
+
+    if not args.no_stats:
+        print(f"validation: {result.validation}", file=sys.stderr)
+        for w in result.warnings:
+            print(f"warning: {w}", file=sys.stderr)
+        if result.fallback:
+            print("original kept (no change was applied)", file=sys.stderr)
+        for r in compare_tokens(source, result.code, DEFAULT_ENCODINGS):
+            print(
+                f"{r.encoding}: {r.before} -> {r.after} ({r.percent:.1f}% saved)", file=sys.stderr
+            )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

llm_compact/core/__init__.py

@@ -0,0 +1,26 @@
+from .cfamily import compact_cfamily
+from .python import compact_python
+from .types import MAX_SOURCE_BYTES, SUPPORTED_LANGUAGES, CompactOptions, CoreResult
+
+__all__ = ["SUPPORTED_LANGUAGES", "CompactOptions", "CoreResult", "compact"]
+
+
+def compact(source: str, language: str, options: CompactOptions | None = None) -> CoreResult:
+    if not isinstance(source, str):
+        raise TypeError("The code must be a string.")
+
+    if language not in SUPPORTED_LANGUAGES:
+        raise ValueError(f"Unsupported language: {language}")
+
+    if len(source.encode("utf-8")) > MAX_SOURCE_BYTES:
+        raise ValueError(f"The code exceeds {MAX_SOURCE_BYTES:,} bytes.")
+
+    if options is not None and not isinstance(options, CompactOptions):
+        raise TypeError("options must be a CompactOptions instance.")
+
+    opt = options or CompactOptions()
+
+    if language == "python":
+        return compact_python(source, opt)
+
+    return compact_cfamily(source, language, opt)