From eadadd86bff3d7b7b01c1d0876c748cc180f59c2 Mon Sep 17 00:00:00 2001 From: hartsock Date: Fri, 5 Jun 2026 20:03:32 -0400 Subject: [PATCH 1/5] Add Chapter 5: CLI Tools (argparse/click -> clap, structured output) Maps argparse/click to clap's derive API: the parser is a type, so typed arguments, ValueEnum choices, and subcommand enums make invalid invocations unrepresentable. Covers structured --json output via serde, doc comments as --help text, exit-code mapping, and in-process CLI testing with try_parse_from. Five "translate this Python" exercises. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 2 + ch05-cli-tools/Cargo.toml | 11 + ch05-cli-tools/README.md | 240 +++++++++++ ch05-cli-tools/exercises/Cargo.toml | 11 + ch05-cli-tools/exercises/src/lib.rs | 393 ++++++++++++++++++ ch05-cli-tools/src/lib.rs | 616 ++++++++++++++++++++++++++++ 6 files changed, 1273 insertions(+) create mode 100644 ch05-cli-tools/Cargo.toml create mode 100644 ch05-cli-tools/README.md create mode 100644 ch05-cli-tools/exercises/Cargo.toml create mode 100644 ch05-cli-tools/exercises/src/lib.rs create mode 100644 ch05-cli-tools/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index a161afd..7cbbc27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,8 @@ members = [ "ch03-traits-and-generics/exercises", "ch04-content-addressable", "ch04-content-addressable/exercises", + "ch05-cli-tools", + "ch05-cli-tools/exercises", ] [workspace.package] diff --git a/ch05-cli-tools/Cargo.toml b/ch05-cli-tools/Cargo.toml new file mode 100644 index 0000000..ac51a58 --- /dev/null +++ b/ch05-cli-tools/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ch05-cli-tools" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Chapter 5: CLI Tools — the parser is the type checker" + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/ch05-cli-tools/README.md b/ch05-cli-tools/README.md new file mode 100644 index 0000000..c711edc --- /dev/null +++ b/ch05-cli-tools/README.md @@ -0,0 +1,240 @@ +# Chapter 5: CLI Tools + +## The Big Idea + +In Python, `argparse` (or `click`) parses the command line into a +`Namespace` — a loosely-typed bag of attributes. Whether `args.port` is an +`int` or a `str` depends on whether you remembered `type=int`. Whether it's +in a valid range depends on a check you wrote somewhere downstream — or +didn't. The parser and your program's expectations are two separate things +that you keep in sync by hand. + +In Rust, the `clap` crate (derive API) inverts this: **the parser IS a +type**. You define a struct, and the struct *is* the command-line interface. +Field types become argument types. Doc comments become `--help` text. +Subcommands are enum variants. By the time parsing succeeds, every value is +already the right type, in the right range, one of the allowed choices. +Invalid input never makes it past the front door — the rest of your program +can't even *represent* a bad configuration. + +This is Chapter 3's lesson (the compiler checks what Python checks at +runtime) applied to a program's outermost boundary: the place where +untrusted strings from a shell become the data your program runs on. + +## Python Analogies + +### `ArgumentParser` = a derive struct + +```python +import argparse + +parser = argparse.ArgumentParser(prog="greet", description="Greet someone") +parser.add_argument("name", help="Name of the person to greet") +parser.add_argument("-c", "--count", type=int, default=1, + help="Number of times to repeat the greeting") +parser.add_argument("-l", "--loud", action="store_true") +args = parser.parse_args() # -> Namespace +``` + +```rust +use clap::Parser; + +/// Greet someone +#[derive(Parser)] +struct Greet { + /// Name of the person to greet + name: String, + + /// Number of times to repeat the greeting + #[arg(short, long, default_value_t = 1)] + count: u8, + + /// Shout the greeting in uppercase + #[arg(short, long)] + loud: bool, +} + +let args = Greet::parse(); // -> Greet, fully typed +``` + +**Key insight:** argparse builds the parser with imperative calls and gives +you back an untyped `Namespace`. clap derives the parser *from a type* and +gives you back that type. There is no gap between "what the parser accepts" +and "what your program expects" — they are the same declaration. + +### `type=int` = the field's type (but enforced everywhere) + +```python +parser.add_argument("--port", type=int, default=8080) +# Forget type=int and args.port is the string "8080". +# Nothing stops --port 99999 or --port -1 either. +``` + +```rust +/// Port to listen on (1024-65535) +#[arg(long, default_value_t = 8080, + value_parser = clap::value_parser!(u16).range(1024..=65535))] +port: u16, +``` + +**Key insight:** `u16` makes negative and oversized ports *unrepresentable* +— not invalid, unrepresentable. The `range()` parser narrows further, and +the rejection happens at parse time with a clear error message. The parser +is the type checker, and validation is part of parsing — not something you +remember to do later. + +### `choices=[...]` = `ValueEnum` + +```python +parser.add_argument("--level", choices=["debug", "info", "warn", "error"]) +# args.level is still a string. Every consumer compares strings, +# and a typo like `if args.level == "wran":` fails silently. +``` + +```rust +#[derive(ValueEnum, Clone, Copy)] +enum LogLevel { Debug, Info, Warn, Error } + +#[arg(long, value_enum, default_value = "info")] +level: LogLevel, +``` + +**Key insight:** the parser converts `"warn"` into `LogLevel::Warn` once, +at the boundary. Downstream code uses `match` — and the compiler insists +every variant is handled. A typo'd level name isn't a silent bug; it +doesn't compile. + +### click groups = subcommand enums + +```python +@click.group() +def tasks(): ... + +@tasks.command() +@click.argument("title") +@click.option("-p", "--priority", type=click.IntRange(1, 5), default=3) +def add(title, priority): ... + +@tasks.command() +@click.argument("id", type=int) +def done(id): ... +``` + +```rust +#[derive(Subcommand)] +enum TaskCommand { + /// Add a new task + Add { + title: String, + #[arg(short, long, default_value_t = 3, + value_parser = clap::value_parser!(u8).range(1..=5))] + priority: u8, + }, + /// Mark a task as done + Done { id: u32 }, +} +``` + +**Key insight:** click scatters a CLI across decorated functions; clap +gathers it into one enum you can read top to bottom. Dispatch is a `match` +— exhaustive, so adding a subcommand forces you to handle it everywhere it +matters. Forgetting one is a compile error, not a runtime surprise. + +### Structured output instead of print soup + +```python +# Anti-pattern: the output IS the format. Scripts that consume this +# tool end up parsing your prose with regexes. +print(f"task {id}: {title} {'(done)' if done else ''}") + +# Better: keep results as data, render at the edge. +task = {"id": id, "title": title, "done": done} +print(json.dumps(task) if args.json else render_text(task)) +``` + +```rust +#[derive(Serialize)] +struct TaskRecord { id: u32, title: String, done: bool } + +fn render_tasks(tasks: &[TaskRecord], format: OutputFormat) -> String { + match format { + OutputFormat::Text => /* human-readable lines */, + OutputFormat::Json => serde_json::to_string(tasks).unwrap(), + } +} +``` + +**Key insight:** same discipline in both languages — compute data, render +once, at the edge — but Rust makes the shape of the output a `Serialize` +type, so `--json` output is guaranteed parseable. A tool that can speak +JSON is a tool other programs (and scripts, and agents) can build on. + +### Help text as a first-class artifact + +The doc comments on the struct and its fields *are* the `--help` text. The +same comment serves rustdoc, your teammates reading the source, and the +user at the terminal. Documentation can't drift from behavior because they +are one declaration. In argparse, `help="..."` strings live apart from the +code that uses the values — and quietly rot. + +### Exit codes + +```python +# argparse exits 2 on bad usage. Your own failures exit with whatever +# you remembered to pass to sys.exit() — often nothing, so a failed +# run exits 0 and the calling shell script merrily continues. +sys.exit(1) +``` + +```rust +enum CliError { + Usage(String), // exit 2: the user invoked the tool incorrectly + Runtime(String), // exit 1: the arguments were fine; the work failed +} + +fn exit_code(result: &Result) -> u8 { + match result { + Ok(_) => 0, + Err(CliError::Runtime(_)) => 1, + Err(CliError::Usage(_)) => 2, + } +} +``` + +**Key insight:** Chapter 2's `Result` reaches all the way to the shell. An +enum of failure kinds maps to exit codes in exactly one place, so `&&` and +`set -e` in calling scripts actually mean something. + +## Testing CLIs Without Spawning Processes + +In Python you'd reach for `subprocess.run([...])` or click's `CliRunner`. +clap parsers have `try_parse_from`, which parses from any slice of strings: + +```rust +let args = Greet::try_parse_from(["greet", "Alice", "--count", "2"]).unwrap(); +assert_eq!(args.count, 2); +``` + +Your entire CLI — parsing, validation, dispatch, rendering, exit-code +mapping — is plain functions over plain data, testable in-process. The only +untested line left is `main` calling `Greet::parse()`. Every test in this +chapter works this way; no test spawns a process. + +## Summary + +| Python | Rust | What Changes | +|--------|------|-------------| +| `ArgumentParser` + `add_argument` | `#[derive(Parser)]` struct | The parser is a type | +| `Namespace` (stringly-typed) | Your struct (fully typed) | No gap between parsed and expected | +| `type=int`, manual range checks | Field types + `value_parser` | Validation at parse time, invalid states unrepresentable | +| `choices=["a", "b"]` | `ValueEnum` enums | Typos become compile errors | +| click groups / `add_subparsers` | `#[derive(Subcommand)]` enums | Exhaustive dispatch via `match` | +| f-string print soup | `Serialize` + `--json` | Output is data; machines can consume it | +| `help="..."` kwargs | Doc comments | Help, docs, and code are one declaration | +| `sys.exit(n)` (if remembered) | `CliError` enum → exit code | Failure taxonomy in one place | +| `CliRunner` / `subprocess` | `try_parse_from` | Test the whole CLI in-process | + +## Next Steps + +Open `src/lib.rs` to see these concepts in working code, then try the +exercises in `exercises/`. diff --git a/ch05-cli-tools/exercises/Cargo.toml b/ch05-cli-tools/exercises/Cargo.toml new file mode 100644 index 0000000..a648be6 --- /dev/null +++ b/ch05-cli-tools/exercises/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ch05-exercises" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Exercises for Chapter 5: CLI Tools" + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/ch05-cli-tools/exercises/src/lib.rs b/ch05-cli-tools/exercises/src/lib.rs new file mode 100644 index 0000000..8ddb4ee --- /dev/null +++ b/ch05-cli-tools/exercises/src/lib.rs @@ -0,0 +1,393 @@ +//! # Chapter 5 Exercises: CLI Tools +//! +//! Each exercise shows a Python snippet and asks you to write the Rust +//! equivalent. Replace the `todo!()` markers with working code. +//! +//! Run tests: `cargo test -p ch05-exercises` + +// These allows are intentional: exercise stubs have unused parameters +// and fields until the student fills in the todo!() markers. +#![allow(unused_variables, dead_code)] + +use clap::{Parser, Subcommand, ValueEnum}; +use serde::Serialize; + +// ============================================================ +// Exercise 1: From Parsed Args to Output +// ============================================================ +// +// Python version: +// ```python +// import argparse +// +// parser = argparse.ArgumentParser(prog="shout") +// parser.add_argument("words", nargs="+") +// parser.add_argument("-r", "--repeat", type=int, default=1) +// args = parser.parse_args() +// +// line = " ".join(args.words).upper() + "!" +// print("\n".join([line] * args.repeat)) +// ``` +// +// The parser struct is provided — read it carefully; the field types and +// attributes ARE the argument definitions. Implement `run_shout`: +// join the words with spaces, uppercase them, append "!", and repeat +// the line `repeat` times joined by newlines. + +/// Shout some words. +#[derive(Parser, Debug)] +#[command(name = "shout")] +pub struct Shout { + /// Words to shout (at least one) + #[arg(required = true)] + pub words: Vec, + + /// Number of times to repeat the line + #[arg(short, long, default_value_t = 1)] + pub repeat: u8, +} + +pub fn run_shout(args: &Shout) -> String { + todo!("Join words with spaces, uppercase, add '!', repeat with newlines") +} + +// ============================================================ +// Exercise 2: The Builder API +// ============================================================ +// +// Python version: +// ```python +// parser = argparse.ArgumentParser(prog="copy") +// parser.add_argument("source", help="File to copy") +// parser.add_argument("dest", help="Where to copy it") +// parser.add_argument("-f", "--force", action="store_true", +// help="Overwrite the destination if it exists") +// ``` +// +// The derive API is sugar over clap's builder API — which looks a lot +// like argparse. Build the same parser by hand: +// - a required positional arg "source" +// - a required positional arg "dest" +// - a "force" flag with short 'f' and long "force" +// +// Hints (use full paths, no extra imports needed): +// clap::Command::new("copy") +// .arg(clap::Arg::new("source").required(true)) +// flags use .action(clap::ArgAction::SetTrue) + +pub fn build_copy_command() -> clap::Command { + todo!("Build the `copy` command with source, dest, and --force") +} + +// ============================================================ +// Exercise 3: Subcommand Dispatch +// ============================================================ +// +// Python version (click): +// ```python +// @click.group() +// def notes(): ... +// +// @notes.command() +// @click.argument("text") +// def add(text): +// click.echo(f"added note: {text}") +// +// @notes.command(name="list") +// @click.option("--limit", type=int) +// def list_(limit): +// if limit is None: +// click.echo("listing all notes") +// else: +// click.echo(f"listing {limit} notes") +// +// @notes.command() +// @click.argument("id", type=int) +// def delete(id): +// click.echo(f"deleted note {id}") +// ``` +// +// The enum is provided. Implement `describe` with a match over the +// variants, producing exactly the strings click would echo above. + +#[derive(Parser, Debug)] +#[command(name = "notes")] +pub struct NotesCli { + #[command(subcommand)] + pub command: NotesCommand, +} + +#[derive(Subcommand, Debug, PartialEq)] +pub enum NotesCommand { + /// Add a note + Add { + /// The note text + text: String, + }, + /// List notes + List { + /// Show at most this many notes + #[arg(long)] + limit: Option, + }, + /// Delete a note + Delete { + /// Note id to delete + id: u32, + }, +} + +pub fn describe(command: &NotesCommand) -> String { + todo!("Match each variant: 'added note: TEXT', 'listing all notes' / 'listing N notes', 'deleted note ID'") +} + +// ============================================================ +// Exercise 4: Structured Output +// ============================================================ +// +// Python version: +// ```python +// def render(packages, as_json): +// if as_json: +// return json.dumps(packages) # list of dicts +// lines = [] +// for p in packages: +// suffix = " (installed)" if p["installed"] else "" +// lines.append(f"{p['name']} {p['version']}{suffix}") +// return "\n".join(lines) +// ``` +// +// Keep results as data; render once, at the edge. Implement `render`: +// - Text: one line per package: "NAME VERSION" plus " (installed)" +// if installed, joined with newlines +// - Json: serde_json::to_string of the whole slice + +#[derive(ValueEnum, Clone, Copy, Debug, PartialEq)] +pub enum OutputFormat { + Text, + Json, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct Package { + pub name: String, + pub version: String, + pub installed: bool, +} + +impl Package { + pub fn new(name: &str, version: &str, installed: bool) -> Self { + Self { + name: name.to_string(), + version: version.to_string(), + installed, + } + } +} + +pub fn render(packages: &[Package], format: OutputFormat) -> String { + todo!("Text: lines like 'name version (installed)'; Json: serde_json::to_string") +} + +// ============================================================ +// Exercise 5: Exit Codes +// ============================================================ +// +// Python version: +// ```python +// import sys +// +// PROTECTED = {"/", ""} +// +// def main(): +// args = parser.parse_args() # argparse exits 2 on bad usage +// if args.target in PROTECTED: +// print("refusing to remove protected path", file=sys.stderr) +// sys.exit(1) # runtime failure +// verb = "would remove" if args.dry_run else "removed" +// print(f"{verb} {args.target}") +// sys.exit(0) +// ``` +// +// Implement `run_remove` and `exit_code`: +// - Parse with `Remove::try_parse_from(argv)`; on a parse error return +// Err(CliError::Usage) (don't worry about --help here) +// - If target is "/" or "" return Err(CliError::Refused) +// - Otherwise Ok("would remove TARGET") when dry_run, Ok("removed TARGET") +// when not +// - exit_code: Ok = 0, Refused = 1, Usage = 2 + +/// Remove a file (pretend). +#[derive(Parser, Debug)] +#[command(name = "remove")] +pub struct Remove { + /// Path to remove + pub target: String, + + /// Show what would happen without doing it + #[arg(long)] + pub dry_run: bool, +} + +#[derive(Debug, PartialEq)] +pub enum CliError { + /// Bad arguments — exit 2. + Usage, + /// Refused to operate on a protected path — exit 1. + Refused, +} + +pub fn run_remove(argv: &[&str]) -> Result { + todo!("Parse argv, refuse protected targets, honor --dry-run") +} + +pub fn exit_code(result: &Result) -> u8 { + todo!("Ok = 0, Refused = 1, Usage = 2") +} + +// ============================================================ +// Tests — do not modify below this line +// ============================================================ + +#[cfg(test)] +mod tests { + use super::*; + + // Exercise 1 + #[test] + fn ex1_shout_one_word() { + let args = Shout::try_parse_from(["shout", "hello"]).unwrap(); + assert_eq!(run_shout(&args), "HELLO!"); + } + + #[test] + fn ex1_shout_joins_words() { + let args = Shout::try_parse_from(["shout", "hello", "world"]).unwrap(); + assert_eq!(run_shout(&args), "HELLO WORLD!"); + } + + #[test] + fn ex1_shout_repeats() { + let args = Shout::try_parse_from(["shout", "-r", "3", "go"]).unwrap(); + assert_eq!(run_shout(&args), "GO!\nGO!\nGO!"); + } + + // Exercise 2 + #[test] + fn ex2_copy_parses_positionals() { + let matches = build_copy_command() + .try_get_matches_from(["copy", "a.txt", "b.txt"]) + .unwrap(); + assert_eq!( + matches.get_one::("source").map(String::as_str), + Some("a.txt") + ); + assert_eq!( + matches.get_one::("dest").map(String::as_str), + Some("b.txt") + ); + assert!(!matches.get_flag("force")); + } + + #[test] + fn ex2_copy_requires_dest() { + assert!(build_copy_command() + .try_get_matches_from(["copy", "only-source.txt"]) + .is_err()); + } + + #[test] + fn ex2_copy_force_flag() { + let long = build_copy_command() + .try_get_matches_from(["copy", "a", "b", "--force"]) + .unwrap(); + assert!(long.get_flag("force")); + + let short = build_copy_command() + .try_get_matches_from(["copy", "a", "b", "-f"]) + .unwrap(); + assert!(short.get_flag("force")); + } + + // Exercise 3 + #[test] + fn ex3_add_describes() { + let cli = NotesCli::try_parse_from(["notes", "add", "buy milk"]).unwrap(); + assert_eq!(describe(&cli.command), "added note: buy milk"); + } + + #[test] + fn ex3_list_without_limit() { + let cli = NotesCli::try_parse_from(["notes", "list"]).unwrap(); + assert_eq!(describe(&cli.command), "listing all notes"); + } + + #[test] + fn ex3_list_with_limit() { + let cli = NotesCli::try_parse_from(["notes", "list", "--limit", "5"]).unwrap(); + assert_eq!(describe(&cli.command), "listing 5 notes"); + } + + #[test] + fn ex3_delete_describes() { + let cli = NotesCli::try_parse_from(["notes", "delete", "12"]).unwrap(); + assert_eq!(describe(&cli.command), "deleted note 12"); + } + + // Exercise 4 + #[test] + fn ex4_text_output() { + let packages = vec![ + Package::new("serde", "1.0.200", true), + Package::new("clap", "4.5.0", false), + ]; + assert_eq!( + render(&packages, OutputFormat::Text), + "serde 1.0.200 (installed)\nclap 4.5.0" + ); + } + + #[test] + fn ex4_json_output_is_parseable() { + let packages = vec![Package::new("serde", "1.0.200", true)]; + let json = render(&packages, OutputFormat::Json); + + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed[0]["name"], "serde"); + assert_eq!(parsed[0]["version"], "1.0.200"); + assert_eq!(parsed[0]["installed"], true); + } + + #[test] + fn ex4_json_empty_list() { + assert_eq!(render(&[], OutputFormat::Json), "[]"); + } + + // Exercise 5 + #[test] + fn ex5_remove_success() { + let result = run_remove(&["remove", "old.log"]); + assert_eq!(result, Ok("removed old.log".to_string())); + assert_eq!(exit_code(&result), 0); + } + + #[test] + fn ex5_dry_run() { + let result = run_remove(&["remove", "old.log", "--dry-run"]); + assert_eq!(result, Ok("would remove old.log".to_string())); + } + + #[test] + fn ex5_refuses_protected_path() { + let result = run_remove(&["remove", "/"]); + assert_eq!(result, Err(CliError::Refused)); + assert_eq!(exit_code(&result), 1); + } + + #[test] + fn ex5_usage_error() { + let result = run_remove(&["remove"]); // missing target + assert_eq!(result, Err(CliError::Usage)); + assert_eq!(exit_code(&result), 2); + } +} diff --git a/ch05-cli-tools/src/lib.rs b/ch05-cli-tools/src/lib.rs new file mode 100644 index 0000000..ff830e2 --- /dev/null +++ b/ch05-cli-tools/src/lib.rs @@ -0,0 +1,616 @@ +//! # Chapter 5: CLI Tools +//! +//! This module maps Python's `argparse`/`click` patterns to Rust's `clap` +//! (derive API). The central idea: in clap, the parser IS a type. Parsing +//! and validation happen in one step, and the rest of your program receives +//! real typed values instead of a stringly-typed namespace. +//! +//! Run the tests: `cargo test -p ch05-cli-tools` + +use clap::{CommandFactory, Parser, Subcommand, ValueEnum}; +use serde::Serialize; + +// --------------------------------------------------------------------------- +// 1. Your first parser — argparse ArgumentParser vs clap derive +// --------------------------------------------------------------------------- + +/// Greet someone from the command line. +/// +/// Python equivalent: +/// ```python +/// import argparse +/// +/// parser = argparse.ArgumentParser(prog="greet", description="Greet someone") +/// parser.add_argument("name", help="Name of the person to greet") +/// parser.add_argument("-c", "--count", type=int, default=1, +/// help="Number of times to repeat the greeting") +/// parser.add_argument("-l", "--loud", action="store_true", +/// help="Shout the greeting in uppercase") +/// args = parser.parse_args() +/// ``` +/// +/// In clap's derive API, the parser IS the struct. Each field becomes an +/// argument, the field type becomes the argument type, and the doc comments +/// become the `--help` text. No separate `add_argument` calls — the struct +/// definition is the single source of truth. +#[derive(Parser, Debug, PartialEq)] +#[command(name = "greet", version = "1.0.0", about = "Greet someone")] +pub struct Greet { + /// Name of the person to greet + pub name: String, + + /// Number of times to repeat the greeting + #[arg(short, long, default_value_t = 1)] + pub count: u8, + + /// Shout the greeting in uppercase + #[arg(short, long)] + pub loud: bool, +} + +/// Build the greeting from parsed arguments. +/// +/// Notice the signature: this takes `&Greet`, not "whatever came off the +/// command line". By the time this function runs, `count` is already a u8 +/// and `loud` is already a bool. There is nothing left to validate. +pub fn greeting(args: &Greet) -> String { + let line = format!("Hello, {}!", args.name); + let line = if args.loud { line.to_uppercase() } else { line }; + vec![line; args.count as usize].join("\n") +} + +// --------------------------------------------------------------------------- +// 2. Typed arguments — the parser IS the type checker +// --------------------------------------------------------------------------- + +/// Serve files from a directory. +/// +/// Python equivalent: +/// ```python +/// parser = argparse.ArgumentParser(prog="serve") +/// parser.add_argument("--port", type=int, default=8080) +/// parser.add_argument("root", nargs="?", default=".") +/// args = parser.parse_args() +/// +/// # argparse gives you a Namespace. args.port is an int *if* you remembered +/// # type=int — otherwise it's a string and you find out deep in your code. +/// # And nothing stops port=99999 or port=-1 from sliding through. +/// if not (1024 <= args.port <= 65535): +/// parser.error("port out of range") # manual, easy to forget +/// ``` +/// +/// In Rust the field type does the work. `u16` already makes 99999 and -1 +/// unrepresentable; the `range()` value parser narrows it further to the +/// unprivileged ports. Invalid input is rejected at the front door — your +/// program logic never sees it. +#[derive(Parser, Debug, PartialEq)] +#[command(name = "serve", about = "Serve files from a directory")] +pub struct Serve { + /// Port to listen on (1024-65535) + #[arg( + short, + long, + default_value_t = 8080, + value_parser = clap::value_parser!(u16).range(1024..=65535) + )] + pub port: u16, + + /// Directory to serve files from + #[arg(default_value = ".")] + pub root: std::path::PathBuf, +} + +// --------------------------------------------------------------------------- +// 3. Choices become enums — invalid states are unrepresentable +// --------------------------------------------------------------------------- + +/// Log levels as a real enum. +/// +/// Python equivalent: +/// ```python +/// parser.add_argument("--level", choices=["debug", "info", "warn", "error"], +/// default="info") +/// # ...but args.level is still a *string*. Every consumer compares +/// # strings, and a typo like `if args.level == "wran":` fails silently. +/// ``` +/// +/// With `ValueEnum`, the parser converts "warn" into `LogLevel::Warn` at +/// parse time. Downstream code matches on the enum — and the compiler +/// rejects a match arm for a level that doesn't exist. +#[derive(ValueEnum, Clone, Copy, Debug, PartialEq)] +pub enum LogLevel { + Debug, + Info, + Warn, + Error, +} + +impl LogLevel { + /// Numeric severity, for threshold comparisons. + /// + /// A match on an enum must be exhaustive: add a `Trace` variant later + /// and this function stops compiling until you handle it. Python's + /// string comparisons would just silently never match. + pub fn severity(&self) -> u8 { + match self { + LogLevel::Debug => 0, + LogLevel::Info => 1, + LogLevel::Warn => 2, + LogLevel::Error => 3, + } + } +} + +/// Configure logging verbosity. +#[derive(Parser, Debug, PartialEq)] +#[command(name = "logdemo")] +pub struct Logging { + /// Minimum level to print + #[arg(long, value_enum, default_value = "info")] + pub level: LogLevel, +} + +// --------------------------------------------------------------------------- +// 4. Subcommands — click groups vs clap subcommand enums +// --------------------------------------------------------------------------- + +/// A tiny task tracker. +/// +/// Python equivalent (click): +/// ```python +/// @click.group() +/// @click.option("--json", "as_json", is_flag=True) +/// def tasks(as_json): ... +/// +/// @tasks.command() +/// @click.argument("title") +/// @click.option("-p", "--priority", type=click.IntRange(1, 5), default=3) +/// def add(title, priority): ... +/// +/// @tasks.command(name="list") +/// @click.option("--all", "show_all", is_flag=True) +/// def list_(show_all): ... +/// +/// @tasks.command() +/// @click.argument("id", type=int) +/// def done(id): ... +/// ``` +/// +/// click models subcommands as decorated functions discovered at runtime. +/// clap models them as an enum: each subcommand is a variant, each variant's +/// fields are that subcommand's arguments. The full command surface of your +/// CLI is one type you can read top to bottom. +#[derive(Parser, Debug)] +#[command(name = "tasks", about = "A tiny task tracker")] +pub struct TasksCli { + /// Emit machine-readable JSON instead of human-readable text + #[arg(long, global = true)] + pub json: bool, + + #[command(subcommand)] + pub command: TaskCommand, +} + +/// One variant per subcommand. Dispatch is a `match` — and it must be +/// exhaustive, so adding a subcommand forces you to handle it everywhere. +#[derive(Subcommand, Debug, PartialEq)] +pub enum TaskCommand { + /// Add a new task + Add { + /// Short description of the task + title: String, + + /// Priority from 1 (highest) to 5 (lowest) + #[arg( + short, + long, + default_value_t = 3, + value_parser = clap::value_parser!(u8).range(1..=5) + )] + priority: u8, + }, + /// List tasks + List { + /// Include completed tasks + #[arg(long)] + all: bool, + }, + /// Mark a task as done + Done { + /// Task id to complete + id: u32, + }, +} + +/// Dispatch a parsed subcommand to a human-readable result. +/// +/// Python equivalent: click calls the decorated function for you. In clap +/// you match on the enum — more explicit, and the compiler guarantees no +/// subcommand is forgotten. +pub fn dispatch(command: &TaskCommand) -> String { + match command { + TaskCommand::Add { title, priority } => { + format!("added {title:?} at priority {priority}") + } + TaskCommand::List { all } => { + if *all { + "listing all tasks".to_string() + } else { + "listing open tasks".to_string() + } + } + TaskCommand::Done { id } => format!("completed task {id}"), + } +} + +// --------------------------------------------------------------------------- +// 5. Structured output — a --json flag instead of print soup +// --------------------------------------------------------------------------- + +/// Output formats a tool can speak. +#[derive(ValueEnum, Clone, Copy, Debug, PartialEq)] +pub enum OutputFormat { + /// Human-readable lines + Text, + /// Machine-readable JSON + Json, +} + +/// One task, as data — not as a pre-formatted string. +/// +/// Python equivalent: +/// ```python +/// # The anti-pattern: print soup. Output IS the format. +/// print(f"task {id}: {title} {'(done)' if done else ''}") +/// +/// # The fix: keep results as data, choose the rendering at the edge. +/// task = {"id": id, "title": title, "done": done} +/// print(json.dumps(task) if args.json else render_text(task)) +/// ``` +/// +/// The same discipline in Rust: compute a `Vec`, then render it +/// once, at the edge, in the format the caller asked for. Scripts and other +/// programs consume `--json`; humans get text. One source of truth. +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct TaskRecord { + pub id: u32, + pub title: String, + pub done: bool, +} + +/// Render tasks in the requested format. +pub fn render_tasks(tasks: &[TaskRecord], format: OutputFormat) -> String { + match format { + OutputFormat::Text => tasks + .iter() + .map(|t| { + let mark = if t.done { "x" } else { " " }; + format!("[{mark}] {} {}", t.id, t.title) + }) + .collect::>() + .join("\n"), + OutputFormat::Json => { + serde_json::to_string(tasks).expect("serialization should not fail for valid types") + } + } +} + +// --------------------------------------------------------------------------- +// 6. Help text as a first-class artifact +// --------------------------------------------------------------------------- + +/// Render the long help for the `greet` tool. +/// +/// Python equivalent: +/// ```python +/// parser.format_help() # help strings passed as help="..." kwargs +/// ``` +/// +/// In clap derive, the doc comments you wrote on the struct and its fields +/// ARE the help text. Documentation and behavior live in one place, so they +/// cannot drift apart — the same comment serves rustdoc, your teammates, +/// and `--help`. +pub fn greet_help() -> String { + Greet::command().render_long_help().to_string() +} + +// --------------------------------------------------------------------------- +// 7. Exit codes — errors a shell can see +// --------------------------------------------------------------------------- + +/// Errors a CLI run can produce, separated by who got it wrong. +/// +/// Python equivalent: +/// ```python +/// # argparse calls sys.exit(2) on bad usage. Your own failures are +/// # whatever you remember to pass to sys.exit() — often nothing, so a +/// # failed run exits 0 and the calling script merrily continues. +/// ``` +/// +/// Convention: 0 = success, 1 = the operation failed, 2 = the user invoked +/// the tool incorrectly. Encoding the distinction in an enum means the +/// mapping to exit codes happens in exactly one place. +#[derive(Debug, PartialEq)] +pub enum CliError { + /// The arguments were invalid — exit code 2. + Usage(String), + /// The arguments were fine but the operation failed — exit code 1. + Runtime(String), +} + +/// Run the greet tool end to end: parse, validate, produce output. +/// +/// Testable by construction: `try_parse_from` parses from a slice instead +/// of the real process arguments, so the whole pipeline runs inside a unit +/// test — no subprocess required. (`Greet::parse()` is the same parser +/// reading the real `argv`; you'd call that in `main`.) +pub fn run_greet(argv: &[&str]) -> Result { + let args = match Greet::try_parse_from(argv) { + Ok(args) => args, + // --help and --version surface as "errors" from try_parse_from, + // but they are successful runs: print and exit 0. + Err(e) + if e.kind() == clap::error::ErrorKind::DisplayHelp + || e.kind() == clap::error::ErrorKind::DisplayVersion => + { + return Ok(e.to_string()); + } + Err(e) => return Err(CliError::Usage(e.to_string())), + }; + + // A rule the type system can't express — checked after parsing, + // reported as a runtime failure rather than a usage error. + if args.name.trim().is_empty() { + return Err(CliError::Runtime("name must not be blank".to_string())); + } + + Ok(greeting(&args)) +} + +/// Map a run result to a process exit code. +/// +/// In `main` you would end with `std::process::exit(exit_code(&result))`. +pub fn exit_code(result: &Result) -> u8 { + match result { + Ok(_) => 0, + Err(CliError::Runtime(_)) => 1, + Err(CliError::Usage(_)) => 2, + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // Your first parser + + #[test] + fn greet_parses_positional_and_defaults() { + let args = Greet::try_parse_from(["greet", "Alice"]).unwrap(); + assert_eq!(args.name, "Alice"); + assert_eq!(args.count, 1); + assert!(!args.loud); + } + + #[test] + fn greet_parses_short_and_long_flags() { + let short = Greet::try_parse_from(["greet", "Bob", "-c", "2", "-l"]).unwrap(); + let long = Greet::try_parse_from(["greet", "Bob", "--count", "2", "--loud"]).unwrap(); + assert_eq!(short, long); + assert_eq!(short.count, 2); + assert!(short.loud); + } + + #[test] + fn greet_missing_name_is_a_parse_error() { + let err = Greet::try_parse_from(["greet"]).unwrap_err(); + assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument); + } + + #[test] + fn greet_rejects_non_numeric_count() { + // argparse without type=int would happily hand you the string "lots". + assert!(Greet::try_parse_from(["greet", "Alice", "--count", "lots"]).is_err()); + } + + #[test] + fn greeting_repeats_and_shouts() { + let args = Greet::try_parse_from(["greet", "Ada", "-c", "2", "--loud"]).unwrap(); + assert_eq!(greeting(&args), "HELLO, ADA!\nHELLO, ADA!"); + } + + // The parser is the type checker + + #[test] + fn serve_defaults_are_typed() { + let args = Serve::try_parse_from(["serve"]).unwrap(); + assert_eq!(args.port, 8080); // a real u16, not a string + assert_eq!(args.root, std::path::PathBuf::from(".")); + } + + #[test] + fn serve_accepts_valid_port() { + let args = Serve::try_parse_from(["serve", "--port", "3000", "/srv"]).unwrap(); + assert_eq!(args.port, 3000); + assert_eq!(args.root, std::path::PathBuf::from("/srv")); + } + + #[test] + fn serve_rejects_out_of_range_port() { + // 99999 doesn't fit in u16, and 80 is below the declared range. + assert!(Serve::try_parse_from(["serve", "--port", "99999"]).is_err()); + assert!(Serve::try_parse_from(["serve", "--port", "80"]).is_err()); + } + + #[test] + fn serve_rejects_non_numeric_port() { + assert!(Serve::try_parse_from(["serve", "--port", "http"]).is_err()); + } + + // Choices become enums + + #[test] + fn level_parses_into_enum_variant() { + let args = Logging::try_parse_from(["logdemo", "--level", "warn"]).unwrap(); + assert_eq!(args.level, LogLevel::Warn); + } + + #[test] + fn level_default_is_info() { + let args = Logging::try_parse_from(["logdemo"]).unwrap(); + assert_eq!(args.level, LogLevel::Info); + } + + #[test] + fn level_rejects_unknown_choice() { + assert!(Logging::try_parse_from(["logdemo", "--level", "loud"]).is_err()); + } + + #[test] + fn severity_orders_levels() { + assert!(LogLevel::Error.severity() > LogLevel::Debug.severity()); + } + + // Subcommands + + #[test] + fn add_subcommand_parses() { + let cli = TasksCli::try_parse_from(["tasks", "add", "write tests", "-p", "1"]).unwrap(); + assert_eq!( + cli.command, + TaskCommand::Add { + title: "write tests".to_string(), + priority: 1, + } + ); + } + + #[test] + fn add_priority_validated_at_parse_time() { + // click.IntRange equivalent — but enforced by the type's parser. + assert!(TasksCli::try_parse_from(["tasks", "add", "x", "--priority", "9"]).is_err()); + } + + #[test] + fn list_subcommand_parses() { + let cli = TasksCli::try_parse_from(["tasks", "list", "--all"]).unwrap(); + assert_eq!(cli.command, TaskCommand::List { all: true }); + } + + #[test] + fn done_subcommand_parses_typed_id() { + let cli = TasksCli::try_parse_from(["tasks", "done", "42"]).unwrap(); + assert_eq!(cli.command, TaskCommand::Done { id: 42 }); + } + + #[test] + fn global_json_flag_works_after_subcommand() { + let cli = TasksCli::try_parse_from(["tasks", "list", "--json"]).unwrap(); + assert!(cli.json); + } + + #[test] + fn dispatch_covers_every_subcommand() { + let cmd = TaskCommand::Add { + title: "ship it".to_string(), + priority: 2, + }; + assert_eq!(dispatch(&cmd), "added \"ship it\" at priority 2"); + assert_eq!( + dispatch(&TaskCommand::List { all: false }), + "listing open tasks" + ); + assert_eq!(dispatch(&TaskCommand::Done { id: 7 }), "completed task 7"); + } + + // Structured output + + #[test] + fn render_text_is_human_readable() { + let tasks = vec![ + TaskRecord { + id: 1, + title: "write chapter".to_string(), + done: true, + }, + TaskRecord { + id: 2, + title: "review chapter".to_string(), + done: false, + }, + ]; + let text = render_tasks(&tasks, OutputFormat::Text); + assert_eq!(text, "[x] 1 write chapter\n[ ] 2 review chapter"); + } + + #[test] + fn render_json_is_machine_readable() { + let tasks = vec![TaskRecord { + id: 1, + title: "write chapter".to_string(), + done: true, + }]; + let json = render_tasks(&tasks, OutputFormat::Json); + + // Another program can parse this back — that's the point. + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed[0]["id"], 1); + assert_eq!(parsed[0]["title"], "write chapter"); + assert_eq!(parsed[0]["done"], true); + } + + // Help text as a first-class artifact + + #[test] + fn doc_comments_become_help_text() { + let help = greet_help(); + assert!(help.contains("Name of the person to greet")); + assert!(help.contains("Number of times to repeat the greeting")); + assert!(help.contains("--loud")); + } + + #[test] + fn subcommands_appear_in_help() { + let help = TasksCli::command().render_long_help().to_string(); + assert!(help.contains("Add a new task")); + assert!(help.contains("Mark a task as done")); + } + + // Exit codes + + #[test] + fn successful_run_exits_zero() { + let result = run_greet(&["greet", "Alice"]); + assert_eq!(result, Ok("Hello, Alice!".to_string())); + assert_eq!(exit_code(&result), 0); + } + + #[test] + fn usage_error_exits_two() { + let result = run_greet(&["greet", "--count", "nope", "Alice"]); + assert!(matches!(result, Err(CliError::Usage(_)))); + assert_eq!(exit_code(&result), 2); + } + + #[test] + fn runtime_error_exits_one() { + let result = run_greet(&["greet", " "]); + assert_eq!( + result, + Err(CliError::Runtime("name must not be blank".to_string())) + ); + assert_eq!(exit_code(&result), 1); + } + + #[test] + fn help_is_success_not_failure() { + let result = run_greet(&["greet", "--help"]); + assert!(result.is_ok()); + assert_eq!(exit_code(&result), 0); + assert!(result.unwrap().contains("Greet someone")); + } +} From 6f24cf33bcac01e9d8b5fd1f75a5613fdcbe732a Mon Sep 17 00:00:00 2001 From: hartsock Date: Fri, 5 Jun 2026 20:04:20 -0400 Subject: [PATCH 2/5] Add Chapter 6: FFI & PyO3 (ctypes/C extensions -> PyO3, maturin) Pure-Rust text-stats core with 20 inline tests; the PyO3 binding layer (pyfunction/pyclass/pymodule, error mapping to ValueError) sits behind an off-by-default 'python' feature so CI never needs a Python interpreter. Exercises practice C-ABI boundaries, conversion shapes, and exception-ready error types in pure Rust. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 2 + ch06-ffi-pyo3/Cargo.toml | 39 ++ ch06-ffi-pyo3/README.md | 222 ++++++++++++ ch06-ffi-pyo3/exercises/Cargo.toml | 12 + ch06-ffi-pyo3/exercises/src/lib.rs | 385 ++++++++++++++++++++ ch06-ffi-pyo3/src/lib.rs | 551 +++++++++++++++++++++++++++++ 6 files changed, 1211 insertions(+) create mode 100644 ch06-ffi-pyo3/Cargo.toml create mode 100644 ch06-ffi-pyo3/README.md create mode 100644 ch06-ffi-pyo3/exercises/Cargo.toml create mode 100644 ch06-ffi-pyo3/exercises/src/lib.rs create mode 100644 ch06-ffi-pyo3/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 7cbbc27..0617b3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,8 @@ members = [ "ch04-content-addressable/exercises", "ch05-cli-tools", "ch05-cli-tools/exercises", + "ch06-ffi-pyo3", + "ch06-ffi-pyo3/exercises", ] [workspace.package] diff --git a/ch06-ffi-pyo3/Cargo.toml b/ch06-ffi-pyo3/Cargo.toml new file mode 100644 index 0000000..71773cf --- /dev/null +++ b/ch06-ffi-pyo3/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "ch06-ffi-pyo3" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Chapter 6: FFI & PyO3 — packaging Rust as a Python module" + +[lib] +# A Python extension module must be a `cdylib`: a C-compatible shared +# library that CPython can dlopen at import time. The `rlib` keeps this +# crate usable as a plain Rust library too — that's what `cargo test` +# links against. Real mixed Rust/Python crates ship with exactly this +# pair of crate types. +crate-type = ["cdylib", "rlib"] + +[features] +# The Python bindings are OFF by default. This is how real mixed crates +# ship: the core logic is pure Rust (testable with plain `cargo test`, +# no Python interpreter anywhere in sight), and the PyO3 glue is an +# optional layer that only maturin/CI wheel builds turn on: +# +# cargo test # pure Rust — what this repo's CI runs +# cargo check --features python # type-check the binding layer +# maturin develop --features python # build + install into a venv +# +# "dep:pyo3" means: enabling the `python` feature pulls in the optional +# pyo3 dependency below. Nothing else does. +python = ["dep:pyo3"] + +[dependencies] +# `optional = true` — pyo3 is only compiled when the `python` feature is on. +# +# - "extension-module" tells pyo3 NOT to link libpython. The Python +# interpreter that imports the module provides the symbols at runtime. +# (Without this, you'd be embedding Python in Rust — a different use case.) +# - "abi3-py39" builds against Python's stable ABI, version 3.9+. One +# wheel works on 3.9, 3.10, 3.11, 3.12, 3.13... instead of one wheel +# per interpreter version. +pyo3 = { version = "0.23", features = ["extension-module", "abi3-py39"], optional = true } diff --git a/ch06-ffi-pyo3/README.md b/ch06-ffi-pyo3/README.md new file mode 100644 index 0000000..919a976 --- /dev/null +++ b/ch06-ffi-pyo3/README.md @@ -0,0 +1,222 @@ +# Chapter 6: FFI & PyO3 + +## The Big Idea + +Python has always known how to call native code — that's why NumPy is fast. +But the traditional routes are painful. `ctypes` makes *you* declare every +argument type and return type by hand; get one wrong and you don't get an +exception, you get garbage bits or a segfault. Hand-written C extensions are +worse: `PyArg_ParseTuple` format strings, manual reference counting, and a +module-initialization dance that has to be exactly right. + +**PyO3 writes the glue for you.** You write ordinary Rust functions and +structs, add `#[pyfunction]`, `#[pyclass]`, and `#[pymodule]` attributes, +and the macros generate everything a C extension needs — argument parsing, +type conversion, reference counting, error propagation. Then **maturin** +compiles it into a wheel that anyone can `pip install` — *without a Rust +toolchain on their machine*. + +That last part is the point of this whole course: Rust's payoff for a +Python team isn't "rewrite the service," it's "ship the hot path as a +module your colleagues import like any other package." + +## Python Analogies + +### The old way: `ctypes` — you write the contract by hand + +```python +import ctypes + +lib = ctypes.CDLL("./libtextstats.so") # you find the library +lib.ffi_add.argtypes = [ctypes.c_int64, ctypes.c_int64] # you declare types +lib.ffi_add.restype = ctypes.c_int64 # you declare the return type +lib.ffi_add(2, 3) # 5 — if you got all of that right + +# Get restype wrong? No error — just wrong numbers. +# Pass a bad pointer/length pair? Best case garbage, worst case segfault. +``` + +The C ABI only speaks pointers and integers. Strings become +pointer-plus-length pairs, and the *caller* is responsible for both being +right. `src/lib.rs` section 1 shows this boundary from the Rust side — +note the `unsafe` keyword and the `# Safety` contract that the compiler +cannot check for you. + +### The new way: PyO3 — the macro writes the contract + +```rust +use pyo3::prelude::*; + +#[pyfunction] +fn add(a: i64, b: i64) -> i64 { + a + b +} + +#[pymodule] +fn textstats(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(add, m)?)?; + Ok(()) +} +``` + +```python +import textstats +textstats.add(2, 3) # 5 +textstats.add("x", "y") # TypeError — at the call site, not a segfault later +``` + +**Key insight:** the type information ctypes made you re-declare in Python +already exists in the Rust signature. PyO3 reads it from there. Wrong types +become a `TypeError` raised immediately — the conversion layer checks every +argument before your Rust code ever runs. + +### Type conversions: the FromPyObject / IntoPyObject mental model + +Arguments convert *from* Python, return values convert *to* Python: + +| Python side | Rust side | Notes | +|-------------|-----------|-------| +| `str` | `&str` / `String` | `&str` borrows — zero-copy read | +| `int` | `i64`, `usize`, ... | range-checked; `OverflowError` if too big | +| `float` | `f64` | | +| `bool` | `bool` | | +| `list[T]` | `Vec` | converts element by element | +| `dict[K, V]` | `HashMap` | | +| `tuple[A, B]` | `(A, B)` | | +| `None` / value | `Option` | `None` ↔ `Option::None` | +| raised exception | `Result` | see error mapping below | + +This is why the chapter's core functions have the signatures they do: +`tally(words: &[String]) -> HashMap` is *already* the shape +of `def tally(words: list[str]) -> dict[str, int]`. Design your Rust API +with these shapes and the binding layer stays one line per function. + +### Error mapping: `Result` becomes a raised exception + +C functions signal failure by returning `-1` or `NULL` and setting `errno` +— and ctypes won't check unless you ask. Forget the check and the error +silently becomes a "valid" value. + +PyO3 uses Rust's `Result` instead, and the failure path is impossible to +ignore on *either* side of the boundary: + +```rust +impl From for PyErr { + fn from(err: NumberError) -> PyErr { + PyValueError::new_err(err.to_string()) + } +} + +#[pyfunction] +fn parse_number(text: &str) -> PyResult { + Ok(parse_flexible_number(text)?) // `?` converts the error +} +``` + +```python +>>> parse_number("abc") +Traceback (most recent call last): +ValueError: not a number: "abc" +``` + +Rust forces the library author to handle the `Err`; Python's exception +machinery forces the caller to notice it. Your error enum's `Display` +string becomes the exception message — write it for the traceback reader. + +### `#[pyclass]`: structs become Python classes + +```rust +#[pyclass(frozen)] // like @dataclass(frozen=True) +struct TextStats { ... } + +#[pymethods] +impl TextStats { + #[getter] + fn words(&self) -> usize { ... } // becomes a read-only property + + fn __repr__(&self) -> String { ... } // dunders work too +} +``` + +From Python it's an ordinary object: `stats.words`, `repr(stats)`. The +data lives in Rust; Python holds a handle. + +## The maturin Workflow + +[maturin](https://github.com/PyO3/maturin) is to PyO3 what `setuptools` +is to C extensions — except it actually handles the hard parts. A minimal +`pyproject.toml`: + +```toml +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "textstats" +requires-python = ">=3.9" + +[tool.maturin] +features = ["pyo3/extension-module"] +``` + +The development loop: + +```bash +pip install maturin +maturin develop # compile + install into the active venv +python -c "import textstats; print(textstats.add(2, 3))" +``` + +And for shipping: + +```bash +maturin build --release # produces a wheel in target/wheels/ +``` + +Because this chapter's crate uses `abi3-py39` (Python's **stable ABI**), +one compiled wheel works on every CPython from 3.9 up — no per-version +rebuilds. Build wheels for each OS/architecture in CI, publish to PyPI, +and your users just run `pip install textstats`. They never see Rust: +no toolchain, no compile step, no `error: linker 'cc' not found`. That is +the distribution story C extensions never quite delivered. + +## Why the Feature Gate? + +Look at this chapter's `Cargo.toml`: pyo3 is `optional = true`, enabled +only by the `python` feature, and the binding module sits behind +`#[cfg(feature = "python")]`. This is deliberate, and it's how real +mixed crates ship: + +- **The core is pure Rust.** `cargo test` runs everywhere — including + this repo's CI — with no Python interpreter in sight. +- **The bindings are a thin shim.** They convert types and map errors; + the logic they wrap is already tested. +- **The wheel build opts in:** `maturin develop --features python`. + +```bash +cargo test -p ch06-ffi-pyo3 # the default: pure Rust +cargo check -p ch06-ffi-pyo3 --features python # type-check the glue +``` + +Keep your logic and your bindings separable and you can test, benchmark, +and reuse the core as a normal Rust crate — the Python package is just +one consumer of it. + +## Summary + +| Python | Rust + PyO3 | What Changes | +|--------|-------------|-------------| +| `ctypes` argtypes/restype | Rust function signatures | Types declared once, checked by the compiler | +| `PyArg_ParseTuple`, refcounting | `#[pyfunction]` | The macro writes the glue | +| C extension classes | `#[pyclass]` + `#[pymethods]` | A struct with attributes | +| `errno`, `NULL` returns | `PyResult` / `From for PyErr` | Errors become real exceptions | +| `setup.py` + compiler flags | `maturin` | One tool: develop, build, publish | +| One wheel per Python version | `abi3` stable ABI | One wheel for 3.9+ | +| Segfault at a distance | `TypeError` at the call site | The boundary checks everything | + +## Next Steps + +Open `src/lib.rs` to see the raw C ABI next to the PyO3 layer, then try +the exercises in `exercises/` — they practice the conversion-friendly +shapes and exception-ready error types that make a Rust core easy to bind. diff --git a/ch06-ffi-pyo3/exercises/Cargo.toml b/ch06-ffi-pyo3/exercises/Cargo.toml new file mode 100644 index 0000000..4a63f4c --- /dev/null +++ b/ch06-ffi-pyo3/exercises/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ch06-exercises" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Exercises for Chapter 6: FFI & PyO3" + +# No dependencies — and deliberately no pyo3. The exercises practice the +# *shapes* PyO3 works with: C-ABI boundaries, conversion-friendly +# signatures, and error types that map cleanly to Python exceptions. +# All of it is pure Rust, so you can solve the chapter without a Python +# interpreter installed. diff --git a/ch06-ffi-pyo3/exercises/src/lib.rs b/ch06-ffi-pyo3/exercises/src/lib.rs new file mode 100644 index 0000000..5896e9f --- /dev/null +++ b/ch06-ffi-pyo3/exercises/src/lib.rs @@ -0,0 +1,385 @@ +//! # Chapter 6 Exercises: FFI & PyO3 +//! +//! These exercises practice the shapes that make Rust code bindable: +//! thin C-ABI boundaries, conversion-friendly signatures, and error +//! types that map cleanly to Python exceptions. Everything is pure Rust +//! — no Python interpreter needed to solve them. +//! +//! Run tests: `cargo test -p ch06-exercises` + +// These allows are intentional: exercise stubs have unused parameters +// and fields until the student fills in the todo!() markers. +#![allow(unused_variables, dead_code)] + +use std::collections::HashMap; +use std::fmt; + +// ============================================================ +// Exercise 1: Keep the Unsafe Boundary Thin +// ============================================================ +// +// The golden rule of FFI: logic lives in safe Rust; the extern wrapper +// only translates. The C-ABI wrapper below is already written — it's +// one line, and it stays one line. Your job is the safe core it calls. +// +// Python version (what a ctypes caller would eventually reach): +// ```python +// def scale_and_clamp(value, factor, max_value): +// """Scale value by factor, clamping the result to [0, max_value].""" +// return min(max(value * factor, 0.0), max_value) +// ``` +// +// Implement `scale_and_clamp`. The tests exercise the safe function; +// the extern wrapper compiles against your implementation. + +pub fn scale_and_clamp(value: f64, factor: f64, max_value: f64) -> f64 { + todo!("Multiply value by factor, then clamp the result between 0.0 and max_value") +} + +/// The C-ABI boundary — provided for you. Notice how little it does: +/// no logic, just a calling-convention change. This is the shape PyO3 +/// generates for you behind `#[pyfunction]`. +#[no_mangle] +pub extern "C" fn ffi_scale_and_clamp(value: f64, factor: f64, max_value: f64) -> f64 { + scale_and_clamp(value, factor, max_value) +} + +// ============================================================ +// Exercise 2: list[tuple] -> dict (the FromPyObject shape) +// ============================================================ +// +// When a Python caller passes `list[tuple[str, int]]` to a #[pyfunction] +// that takes `Vec<(String, i64)>`, PyO3 converts element by element. +// This exercise is that conversion's classic consumer: Python's +// dict() constructor. +// +// Python version: +// ```python +// def from_pairs(pairs: list[tuple[str, int]]) -> dict[str, int]: +// return dict(pairs) # later duplicates win +// ``` +// +// Build a HashMap from the pairs. If a key appears more than once, +// the LAST occurrence wins — exactly like dict(pairs) in Python. + +pub fn from_pairs(pairs: Vec<(String, i64)>) -> HashMap { + todo!("Insert each pair into a HashMap; later duplicates overwrite earlier ones") +} + +// ============================================================ +// Exercise 3: dict.get — None Becomes Option +// ============================================================ +// +// Python's dict.get returns None for missing keys; Rust's HashMap::get +// returns Option. PyO3 maps between them automatically: a #[pyfunction] +// returning Option gives Python `int | None`. +// +// Python version: +// ```python +// def lookup(table: dict[str, int], key: str) -> int | None: +// return table.get(key) +// +// def lookup_or(table: dict[str, int], key: str, default: int) -> int: +// return table.get(key, default) +// ``` +// +// Implement both. Hint: `HashMap::get` returns `Option<&i64>` — you'll +// need `.copied()` to turn it into `Option`. + +pub fn lookup(table: &HashMap, key: &str) -> Option { + todo!("Return Some(value) if key exists, None otherwise") +} + +pub fn lookup_or(table: &HashMap, key: &str, default: i64) -> i64 { + todo!("Return the value for key, or default if missing — try unwrap_or") +} + +// ============================================================ +// Exercise 4: An Error Type That Maps to ValueError +// ============================================================ +// +// A binding layer converts Rust errors into Python exceptions, and the +// Display string becomes the exception message — it's what Python users +// see in their traceback. Write messages worthy of that. +// +// Python version: +// ```python +// def parse_percent(text: str) -> float: +// text = text.strip() +// if not text: +// raise ValueError("empty input: expected a percentage") +// if not text.endswith("%"): +// raise ValueError(f"missing % sign: '{text}'") +// try: +// return float(text[:-1]) / 100.0 +// except ValueError: +// raise ValueError(f"not a number: '{text[:-1]}'") +// ``` +// +// 1. Implement Display for PercentError (messages must match the tests) +// 2. Implement parse_percent: "85%" -> 0.85, "12.5%" -> 0.125 + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PercentError { + /// Input was empty or only whitespace. + Empty, + /// Input didn't end with '%'; carries the trimmed input. + MissingSign(String), + /// The part before '%' wasn't a number; carries that part. + BadNumber(String), +} + +impl fmt::Display for PercentError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + todo!("Match each variant — see the Python messages above and the tests below") + } +} + +impl std::error::Error for PercentError {} + +pub fn parse_percent(text: &str) -> Result { + todo!("Trim, check for emptiness, require a trailing '%', parse, divide by 100") +} + +// ============================================================ +// Exercise 5: A Struct Shaped Like a #[pyclass] +// ============================================================ +// +// A #[pyclass] is just a Rust struct whose methods make sense from +// Python: a constructor, mutating methods, and queries. Build the +// struct; in a real binding crate you'd add #[pyclass]/#[pymethods] +// and it would ship to Python unchanged. +// +// Python version: +// ```python +// class WordTally: +// def __init__(self): +// self._counts = {} +// +// def add(self, word: str) -> None: +// w = word.lower() +// self._counts[w] = self._counts.get(w, 0) + 1 +// +// def count(self, word: str) -> int: +// return self._counts.get(word.lower(), 0) +// +// def total(self) -> int: +// return sum(self._counts.values()) +// +// def most_common(self, n: int) -> list[tuple[str, int]]: +// ranked = sorted(self._counts.items(), key=lambda kv: (-kv[1], kv[0])) +// return ranked[:n] +// ``` +// +// Implement the four methods. most_common sorts by count (descending), +// breaking ties alphabetically — deterministic output, always. + +#[derive(Debug, Default)] +pub struct WordTally { + counts: HashMap, +} + +impl WordTally { + pub fn new() -> Self { + Self::default() + } + + /// Record one occurrence of a word (case-insensitive). + pub fn add(&mut self, word: &str) { + todo!("Lowercase the word and increment its count") + } + + /// How many times has this word been added (case-insensitive)? + pub fn count(&self, word: &str) -> usize { + todo!("Look up the lowercased word; missing words count as 0") + } + + /// Total number of words added. + pub fn total(&self) -> usize { + todo!("Sum all the counts") + } + + /// The n most frequent words: count descending, ties alphabetical. + pub fn most_common(&self, n: usize) -> Vec<(String, usize)> { + todo!("Collect into a Vec, sort by (count desc, word asc), truncate to n") + } +} + +// ============================================================ +// Tests — do not modify below this line +// ============================================================ + +#[cfg(test)] +mod tests { + use super::*; + + // Exercise 1 + #[test] + fn ex1_scales() { + assert!((scale_and_clamp(2.0, 3.0, 100.0) - 6.0).abs() < f64::EPSILON); + } + + #[test] + fn ex1_clamps_high() { + assert!((scale_and_clamp(50.0, 3.0, 100.0) - 100.0).abs() < f64::EPSILON); + } + + #[test] + fn ex1_clamps_low() { + assert!(scale_and_clamp(-5.0, 2.0, 100.0).abs() < f64::EPSILON); + } + + // Exercise 2 + #[test] + fn ex2_builds_map() { + let pairs = vec![("a".to_string(), 1), ("b".to_string(), 2)]; + let map = from_pairs(pairs); + assert_eq!(map.get("a"), Some(&1)); + assert_eq!(map.get("b"), Some(&2)); + assert_eq!(map.len(), 2); + } + + #[test] + fn ex2_last_duplicate_wins() { + let pairs = vec![("k".to_string(), 1), ("k".to_string(), 99)]; + let map = from_pairs(pairs); + assert_eq!(map.get("k"), Some(&99)); + assert_eq!(map.len(), 1); + } + + #[test] + fn ex2_empty_pairs() { + assert!(from_pairs(vec![]).is_empty()); + } + + // Exercise 3 + #[test] + fn ex3_lookup_present() { + let mut table = HashMap::new(); + table.insert("answer".to_string(), 42); + assert_eq!(lookup(&table, "answer"), Some(42)); + } + + #[test] + fn ex3_lookup_missing_is_none() { + let table = HashMap::new(); + assert_eq!(lookup(&table, "missing"), None); + } + + #[test] + fn ex3_lookup_or_default() { + let mut table = HashMap::new(); + table.insert("answer".to_string(), 42); + assert_eq!(lookup_or(&table, "answer", 0), 42); + assert_eq!(lookup_or(&table, "missing", -1), -1); + } + + // Exercise 4 + #[test] + fn ex4_parses_whole_percent() { + let value = parse_percent("85%").unwrap(); + assert!((value - 0.85).abs() < 1e-12); + } + + #[test] + fn ex4_parses_fractional_percent() { + let value = parse_percent("12.5%").unwrap(); + assert!((value - 0.125).abs() < 1e-12); + } + + #[test] + fn ex4_trims_whitespace() { + let value = parse_percent(" 100% ").unwrap(); + assert!((value - 1.0).abs() < 1e-12); + } + + #[test] + fn ex4_empty_is_error() { + assert_eq!(parse_percent(" "), Err(PercentError::Empty)); + } + + #[test] + fn ex4_missing_sign_is_error() { + assert_eq!( + parse_percent("85"), + Err(PercentError::MissingSign("85".to_string())) + ); + } + + #[test] + fn ex4_bad_number_is_error() { + assert_eq!( + parse_percent("abc%"), + Err(PercentError::BadNumber("abc".to_string())) + ); + } + + #[test] + fn ex4_messages_match_python_exceptions() { + // These are the strings Python users would see as str(exc). + assert_eq!( + PercentError::Empty.to_string(), + "empty input: expected a percentage" + ); + assert_eq!( + PercentError::MissingSign("85".to_string()).to_string(), + "missing % sign: '85'" + ); + assert_eq!( + PercentError::BadNumber("abc".to_string()).to_string(), + "not a number: 'abc'" + ); + } + + // Exercise 5 + #[test] + fn ex5_add_and_count() { + let mut tally = WordTally::new(); + tally.add("rust"); + tally.add("Rust"); + tally.add("python"); + assert_eq!(tally.count("RUST"), 2); + assert_eq!(tally.count("python"), 1); + assert_eq!(tally.count("missing"), 0); + } + + #[test] + fn ex5_total() { + let mut tally = WordTally::new(); + tally.add("a"); + tally.add("b"); + tally.add("a"); + assert_eq!(tally.total(), 3); + } + + #[test] + fn ex5_most_common_ranks_by_count() { + let mut tally = WordTally::new(); + for word in ["apple", "banana", "apple", "cherry", "apple", "banana"] { + tally.add(word); + } + assert_eq!( + tally.most_common(2), + vec![("apple".to_string(), 3), ("banana".to_string(), 2)] + ); + } + + #[test] + fn ex5_most_common_breaks_ties_alphabetically() { + let mut tally = WordTally::new(); + for word in ["beta", "alpha", "beta", "alpha"] { + tally.add(word); + } + assert_eq!( + tally.most_common(2), + vec![("alpha".to_string(), 2), ("beta".to_string(), 2)] + ); + } + + #[test] + fn ex5_empty_tally() { + let tally = WordTally::new(); + assert_eq!(tally.total(), 0); + assert!(tally.most_common(5).is_empty()); + } +} diff --git a/ch06-ffi-pyo3/src/lib.rs b/ch06-ffi-pyo3/src/lib.rs new file mode 100644 index 0000000..77eee1d --- /dev/null +++ b/ch06-ffi-pyo3/src/lib.rs @@ -0,0 +1,551 @@ +//! # Chapter 6: FFI & PyO3 +//! +//! This module shows the two ways Rust code reaches Python: the raw C ABI +//! (what `ctypes` calls — manual, fragile, segfault-prone) and PyO3 +//! (declarative bindings where a macro writes the glue for you). +//! +//! The architecture here is the one real mixed crates use: the core logic +//! is **pure Rust** with thorough tests and zero Python knowledge. The +//! PyO3 binding layer lives at the bottom of this file behind +//! `#[cfg(feature = "python")]` — off by default, so `cargo test` never +//! needs a Python interpreter. +//! +//! Run the tests: `cargo test -p ch06-ffi-pyo3` +//! Type-check the bindings: `cargo check -p ch06-ffi-pyo3 --features python` + +use std::collections::{HashMap, HashSet}; +use std::fmt; + +// --------------------------------------------------------------------------- +// 1. The hard way: the raw C ABI (what ctypes sees) +// --------------------------------------------------------------------------- + +/// A function exported with the C calling convention. +/// +/// This is the *only* kind of function `ctypes` can call. From Python: +/// ```python +/// import ctypes +/// lib = ctypes.CDLL("./target/release/libch06_ffi_pyo3.so") +/// lib.ffi_add.argtypes = [ctypes.c_int64, ctypes.c_int64] +/// lib.ffi_add.restype = ctypes.c_int64 +/// lib.ffi_add(2, 3) # 5 +/// ``` +/// +/// Notice everything YOU have to get right: the library path, the argument +/// types, the return type. Get `restype` wrong and ctypes happily +/// reinterprets the bits — no error, just garbage numbers. +/// +/// `#[no_mangle]` keeps the symbol name `ffi_add` instead of a mangled +/// Rust name, and `extern "C"` uses the C calling convention. +#[no_mangle] +pub extern "C" fn ffi_add(a: i64, b: i64) -> i64 { + a + b +} + +/// The C ABI has no `&str`, no `Vec`, no `Option` — only pointers and +/// integers. Passing text means passing a raw pointer and a length, and +/// trusting the caller got both right. +/// +/// From Python, the caller side looks like this: +/// ```python +/// data = b"hello world, hello ffi" +/// lib.ffi_count_spaces.argtypes = [ctypes.c_char_p, ctypes.c_size_t] +/// lib.ffi_count_spaces.restype = ctypes.c_uint64 +/// lib.ffi_count_spaces(data, len(data)) # 3 +/// # Pass len(data) + 1000 instead and you read past the buffer. +/// # Best case: garbage. Worst case: segfault. ctypes won't stop you. +/// ``` +/// +/// # Safety +/// +/// `ptr` must be non-null and point to `len` initialized bytes that remain +/// valid for the duration of the call. The Rust compiler cannot check this +/// — that's exactly the contract C extensions have always made you uphold +/// by hand, and exactly what PyO3 automates away. +#[no_mangle] +pub unsafe extern "C" fn ffi_count_spaces(ptr: *const u8, len: usize) -> u64 { + if ptr.is_null() { + return 0; + } + let bytes = std::slice::from_raw_parts(ptr, len); + bytes.iter().filter(|&&b| b == b' ').count() as u64 +} + +// --------------------------------------------------------------------------- +// 2. The pure-Rust core: a small text-statistics library +// --------------------------------------------------------------------------- +// +// Everything from here down to the `python` module is ordinary Rust. +// No FFI, no unsafe, no Python types. This is the part you test hard, +// because it's the part that does the actual work. The binding layer +// below is thin enough to verify by inspection. + +/// Statistics about a piece of text. +/// +/// Python equivalent (what the binding layer will expose): +/// ```python +/// @dataclass(frozen=True) +/// class TextStats: +/// lines: int +/// words: int +/// chars: int +/// unique_words: int +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TextStats { + pub lines: usize, + pub words: usize, + pub chars: usize, + pub unique_words: usize, +} + +/// Analyze a piece of text. +/// +/// Python equivalent: +/// ```python +/// def analyze(text: str) -> TextStats: +/// words = text.split() +/// return TextStats( +/// lines=len(text.splitlines()), +/// words=len(words), +/// chars=len(text), +/// unique_words=len({w.lower() for w in words}), +/// ) +/// ``` +pub fn analyze(text: &str) -> TextStats { + let words: Vec<&str> = text.split_whitespace().collect(); + let unique: HashSet = words.iter().map(|w| w.to_lowercase()).collect(); + TextStats { + lines: text.lines().count(), + words: words.len(), + chars: text.chars().count(), + unique_words: unique.len(), + } +} + +/// The `n` most frequent words, most frequent first. +/// +/// Python equivalent: +/// ```python +/// from collections import Counter +/// def top_words(text: str, n: int) -> list[tuple[str, int]]: +/// words = [w.strip(string.punctuation).lower() for w in text.split()] +/// return Counter(w for w in words if w).most_common(n) +/// ``` +/// +/// Ties are broken alphabetically so the output is deterministic — +/// `Counter.most_common` leaves tie order unspecified; we do better. +pub fn top_words(text: &str, n: usize) -> Vec<(String, usize)> { + let mut counts: HashMap = HashMap::new(); + for raw in text.split_whitespace() { + let cleaned = raw + .trim_matches(|c: char| !c.is_alphanumeric()) + .to_lowercase(); + if !cleaned.is_empty() { + *counts.entry(cleaned).or_insert(0) += 1; + } + } + let mut pairs: Vec<(String, usize)> = counts.into_iter().collect(); + pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + pairs.truncate(n); + pairs +} + +/// Count occurrences of each word in a list. +/// +/// Python equivalent: +/// ```python +/// def tally(words: list[str]) -> dict[str, int]: +/// counts = {} +/// for w in words: +/// counts[w] = counts.get(w, 0) + 1 +/// return counts +/// ``` +/// +/// Note the *shapes* here: this function takes a slice and returns a +/// `HashMap`. When PyO3 wraps it, the Python caller passes a `list[str]` +/// and gets back a `dict[str, int]` — the conversions are automatic. +pub fn tally(words: &[String]) -> HashMap { + let mut counts = HashMap::new(); + for w in words { + *counts.entry(w.clone()).or_insert(0) += 1; + } + counts +} + +// --------------------------------------------------------------------------- +// 3. Errors the Python side can understand +// --------------------------------------------------------------------------- + +/// Things that can go wrong when parsing a number from messy input. +/// +/// Python equivalent: there isn't one, really. Python functions just +/// `raise ValueError(...)`. In Rust we name each failure mode, and the +/// binding layer maps the whole enum to `ValueError` (section 4). +/// +/// Contrast this with the C-extension world: a C function signals failure +/// by returning -1 or NULL and setting `errno` — and ctypes won't even +/// check *that* unless you ask. Forget the check and the error silently +/// becomes a "valid" value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NumberError { + /// The input was empty or only whitespace. + Empty, + /// The input wasn't a number; carries the offending text. + Invalid(String), +} + +impl fmt::Display for NumberError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + NumberError::Empty => write!(f, "empty input: nothing to parse"), + NumberError::Invalid(s) => write!(f, "not a number: {s:?}"), + } + } +} + +impl std::error::Error for NumberError {} + +/// Parse a number the way spreadsheet exports write them: surrounding +/// whitespace and thousands separators allowed. +/// +/// Python equivalent: +/// ```python +/// def parse_flexible_number(text: str) -> float: +/// cleaned = text.strip().replace(",", "") +/// if not cleaned: +/// raise ValueError("empty input: nothing to parse") +/// return float(cleaned) # raises ValueError on bad input +/// ``` +/// +/// The Rust version returns `Result` instead of raising. The caller MUST +/// handle the error case — there is no "forgot the try/except" failure mode. +pub fn parse_flexible_number(input: &str) -> Result { + let trimmed = input.trim(); + if trimmed.is_empty() { + return Err(NumberError::Empty); + } + let cleaned: String = trimmed.chars().filter(|&c| c != ',').collect(); + cleaned + .parse::() + .map_err(|_| NumberError::Invalid(trimmed.to_string())) +} + +// --------------------------------------------------------------------------- +// 4. The PyO3 binding layer — feature-gated, off by default +// --------------------------------------------------------------------------- +// +// This module only exists when you build with `--features python`. +// The default build (and this repo's CI) never compiles it, never links +// libpython, and never needs a Python interpreter. That separation is +// the whole architecture: a pure-Rust core anyone can `cargo test`, plus +// an optional shim that maturin compiles into a wheel. +// +// Compare the amount of code here with what a hand-written C extension +// needs — PyArg_ParseTuple format strings, manual refcounting with +// Py_INCREF/Py_DECREF, a PyMethodDef table, a PyModuleDef struct, an +// init function... PyO3's macros generate all of it from plain signatures. + +#[cfg(feature = "python")] +mod python { + use super::{NumberError, TextStats}; + use pyo3::exceptions::PyValueError; + use pyo3::prelude::*; + use std::collections::HashMap; + + // -- Error mapping: Rust Err -> Python exception -------------------- + // + // `#[pyfunction]`s return PyResult. The `?` operator converts our + // NumberError into a PyErr via this From impl, and PyO3 raises it as + // a real Python exception on the other side of the boundary: + // + // >>> parse_number("abc") + // Traceback (most recent call last): + // ValueError: not a number: "abc" + // + // No errno. No silently-ignored return codes. Rust's "you must handle + // the Err" becomes Python's "an exception was raised". + impl From for PyErr { + fn from(err: NumberError) -> PyErr { + PyValueError::new_err(err.to_string()) + } + } + + // -- A #[pyclass]: a Rust struct Python can hold -------------------- + // + // This wrapper turns the pure-Rust TextStats into a Python object. + // `name = "TextStats"` is what Python sees; `frozen` makes it + // immutable, like @dataclass(frozen=True). + #[pyclass(name = "TextStats", frozen)] + struct PyTextStats { + inner: TextStats, + } + + #[pymethods] + impl PyTextStats { + // #[getter] methods become read-only properties: stats.words + #[getter] + fn lines(&self) -> usize { + self.inner.lines + } + + #[getter] + fn words(&self) -> usize { + self.inner.words + } + + #[getter] + fn chars(&self) -> usize { + self.inner.chars + } + + #[getter] + fn unique_words(&self) -> usize { + self.inner.unique_words + } + + // Dunder methods work too — this is Python's repr(stats). + fn __repr__(&self) -> String { + format!( + "TextStats(lines={}, words={}, chars={}, unique_words={})", + self.inner.lines, self.inner.words, self.inner.chars, self.inner.unique_words + ) + } + } + + // -- #[pyfunction]s: thin wrappers over the pure-Rust core ---------- + // + // Each wrapper is one or two lines. The type conversions happen in + // the generated glue: + // + // Python str -> &str (borrowed, zero-copy read) + // Python list[str] -> Vec + // Rust HashMap -> Python dict + // Rust Vec<(S, u)> -> Python list[tuple[str, int]] + // Rust Err(e) -> raised exception + // + // Pass a list of ints where list[str] is expected and you get a + // TypeError at the call site — not a segfault three frames later. + + /// analyze(text) -> TextStats + #[pyfunction] + fn analyze(text: &str) -> PyTextStats { + PyTextStats { + inner: super::analyze(text), + } + } + + /// top_words(text, n=10) -> list[tuple[str, int]] + /// + /// The signature attribute gives Python callers a default argument, + /// exactly like `def top_words(text, n=10)`. + #[pyfunction] + #[pyo3(signature = (text, n = 10))] + fn top_words(text: &str, n: usize) -> Vec<(String, usize)> { + super::top_words(text, n) + } + + /// tally(words) -> dict[str, int] + #[pyfunction] + fn tally(words: Vec) -> HashMap { + super::tally(&words) + } + + /// parse_number(text) -> float, raising ValueError on bad input. + #[pyfunction] + fn parse_number(text: &str) -> PyResult { + // `?` converts NumberError -> PyErr via the From impl above. + Ok(super::parse_flexible_number(text)?) + } + + // -- The #[pymodule]: what `import ch06_ffi_pyo3` finds ------------- + // + // The function name must match the compiled module's import name. + // maturin builds the cdylib, names it correctly for the platform + // (.so / .pyd), and packages it as a wheel. + #[pymodule] + fn ch06_ffi_pyo3(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(analyze, m)?)?; + m.add_function(wrap_pyfunction!(top_words, m)?)?; + m.add_function(wrap_pyfunction!(tally, m)?)?; + m.add_function(wrap_pyfunction!(parse_number, m)?)?; + m.add_class::()?; + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Tests — pure Rust, no Python interpreter required +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // The C ABI layer + + #[test] + fn ffi_add_works() { + assert_eq!(ffi_add(2, 3), 5); + assert_eq!(ffi_add(-10, 4), -6); + } + + #[test] + fn ffi_count_spaces_works() { + let data = b"hello world, hello ffi"; + // SAFETY: pointer and length both come from the same live slice. + let count = unsafe { ffi_count_spaces(data.as_ptr(), data.len()) }; + assert_eq!(count, 3); + } + + #[test] + fn ffi_count_spaces_null_is_zero() { + // SAFETY: the function documents that null is checked and returns 0. + let count = unsafe { ffi_count_spaces(std::ptr::null(), 0) }; + assert_eq!(count, 0); + } + + #[test] + fn ffi_count_spaces_short_len_undercounts() { + // This is the ctypes trap in miniature: the length is the contract. + // Pass a smaller len and you silently get a different answer. + let data = b"a b c"; + // SAFETY: 3 <= data.len(), so the read stays in bounds. + let count = unsafe { ffi_count_spaces(data.as_ptr(), 3) }; + assert_eq!(count, 1); // only "a b" was visible + } + + // The pure-Rust core: analyze + + #[test] + fn analyze_counts_everything() { + let stats = analyze("the quick brown fox\njumps over the lazy dog"); + assert_eq!(stats.lines, 2); + assert_eq!(stats.words, 9); + assert_eq!(stats.chars, 43); + assert_eq!(stats.unique_words, 8); // "the" appears twice + } + + #[test] + fn analyze_empty_text() { + let stats = analyze(""); + assert_eq!( + stats, + TextStats { + lines: 0, + words: 0, + chars: 0, + unique_words: 0, + } + ); + } + + #[test] + fn analyze_unique_is_case_insensitive() { + let stats = analyze("Rust rust RUST"); + assert_eq!(stats.words, 3); + assert_eq!(stats.unique_words, 1); + } + + #[test] + fn analyze_counts_chars_not_bytes() { + // Python's len("héllo") is 5; Rust's "héllo".len() is 6 (bytes). + // We count chars so the Rust answer matches Python's intuition. + let stats = analyze("héllo"); + assert_eq!(stats.chars, 5); + } + + // top_words + + #[test] + fn top_words_orders_by_frequency() { + let words = top_words("apple banana apple cherry apple banana", 2); + assert_eq!( + words, + vec![("apple".to_string(), 3), ("banana".to_string(), 2)] + ); + } + + #[test] + fn top_words_breaks_ties_alphabetically() { + let words = top_words("beta alpha beta alpha", 2); + assert_eq!( + words, + vec![("alpha".to_string(), 2), ("beta".to_string(), 2)] + ); + } + + #[test] + fn top_words_strips_punctuation_and_case() { + let words = top_words("Hello, hello! HELLO?", 1); + assert_eq!(words, vec![("hello".to_string(), 3)]); + } + + #[test] + fn top_words_empty_text() { + assert!(top_words("", 5).is_empty()); + } + + // tally + + #[test] + fn tally_counts_occurrences() { + let words = vec!["a".to_string(), "b".to_string(), "a".to_string()]; + let counts = tally(&words); + assert_eq!(counts.get("a"), Some(&2)); + assert_eq!(counts.get("b"), Some(&1)); + assert_eq!(counts.len(), 2); + } + + #[test] + fn tally_empty_list() { + assert!(tally(&[]).is_empty()); + } + + // parse_flexible_number + error mapping + + #[test] + fn parse_plain_number() { + let value = parse_flexible_number("42.5").unwrap(); + assert!((value - 42.5).abs() < f64::EPSILON); + } + + #[test] + fn parse_with_whitespace_and_commas() { + let value = parse_flexible_number(" 1,234.56 ").unwrap(); + assert!((value - 1234.56).abs() < f64::EPSILON); + } + + #[test] + fn parse_negative() { + let value = parse_flexible_number("-3.5").unwrap(); + assert!((value + 3.5).abs() < f64::EPSILON); + } + + #[test] + fn parse_empty_is_error() { + assert_eq!(parse_flexible_number(" "), Err(NumberError::Empty)); + } + + #[test] + fn parse_garbage_is_error() { + assert_eq!( + parse_flexible_number("abc"), + Err(NumberError::Invalid("abc".to_string())) + ); + } + + #[test] + fn error_messages_read_like_python_exceptions() { + // These Display strings are what Python users will see as str(exc) + // once the binding layer maps NumberError -> ValueError. + assert_eq!( + NumberError::Empty.to_string(), + "empty input: nothing to parse" + ); + assert_eq!( + NumberError::Invalid("abc".to_string()).to_string(), + "not a number: \"abc\"" + ); + } +} From 74d5c3187b2478b3a725cb678a23535757088175 Mon Sep 17 00:00:00 2001 From: hartsock Date: Fri, 5 Jun 2026 20:06:45 -0400 Subject: [PATCH 3/5] Add Chapter 7: PyO3 House Style (feature-gated bindings, error mapping, zero-copy) Operationalizes the Rust-core-with-Python-reach style: pyo3 sits behind a default-off 'python' feature so cargo test stays pure Rust (the CI constraint is the lesson), one error enum maps exhaustively to typed Python exceptions, and zero-copy parsing shows what crosses the FFI line. 19 passing example tests; 5 exercises (19 tests) covering error design, exception mapping, borrow-don't-clone, and Pythonic API shape. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 2 + ch07-pyo3-house-style/Cargo.toml | 24 + ch07-pyo3-house-style/README.md | 215 +++++++ ch07-pyo3-house-style/exercises/Cargo.toml | 11 + ch07-pyo3-house-style/exercises/src/lib.rs | 412 ++++++++++++ ch07-pyo3-house-style/src/lib.rs | 714 +++++++++++++++++++++ 6 files changed, 1378 insertions(+) create mode 100644 ch07-pyo3-house-style/Cargo.toml create mode 100644 ch07-pyo3-house-style/README.md create mode 100644 ch07-pyo3-house-style/exercises/Cargo.toml create mode 100644 ch07-pyo3-house-style/exercises/src/lib.rs create mode 100644 ch07-pyo3-house-style/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 0617b3d..2893935 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ members = [ "ch05-cli-tools/exercises", "ch06-ffi-pyo3", "ch06-ffi-pyo3/exercises", + "ch07-pyo3-house-style", + "ch07-pyo3-house-style/exercises", ] [workspace.package] diff --git a/ch07-pyo3-house-style/Cargo.toml b/ch07-pyo3-house-style/Cargo.toml new file mode 100644 index 0000000..72281f4 --- /dev/null +++ b/ch07-pyo3-house-style/Cargo.toml @@ -0,0 +1,24 @@ +# The house style in one Cargo.toml: +# +# - pyo3 is OPTIONAL. Plain `cargo build` / `cargo test` never compile it, +# so crates.io users (and this repo's CI) pay nothing for the Python face. +# - The `python` feature turns on the binding layer at the bottom of +# src/lib.rs: `cargo check --features python`. +# - abi3-py39 builds one wheel that works on every CPython >= 3.9 instead +# of one wheel per Python version. +# - A real published project would also set crate-type = ["cdylib", "rlib"] +# and build wheels with maturin; we omit that here so this teaching crate +# stays a plain Rust library. + +[package] +name = "ch07-pyo3-house-style" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Chapter 7: PyO3 House Style — feature-gated bindings, error mapping, zero-copy" + +[dependencies] +pyo3 = { version = "0.23", optional = true, features = ["extension-module", "abi3-py39"] } + +[features] +python = ["dep:pyo3"] diff --git a/ch07-pyo3-house-style/README.md b/ch07-pyo3-house-style/README.md new file mode 100644 index 0000000..79159bd --- /dev/null +++ b/ch07-pyo3-house-style/README.md @@ -0,0 +1,215 @@ +# Chapter 7: PyO3 House Style + +## The Big Idea + +Chapter 6 introduced the two-layer pattern: a pure-Rust core plus a thin +PyO3 boundary. This chapter turns that pattern into a **house style** — the +conventions for shipping a Rust core with Python reach. One codebase, one +source of truth, two ecosystems served: a crate on crates.io and a wheel +on PyPI, with neither audience treated as second-class. + +The style is five rules. Each one answers the same question from the +roadmap: *how should this feel from Python?* — without ever letting the +answer leak into the Rust core. + +## Rule 1: The Core Never Mentions Python + +The core logic is a plain Rust crate. The bindings are a skin, gated +behind a default-off feature: + +```toml +[dependencies] +pyo3 = { version = "0.23", optional = true, features = ["extension-module", "abi3-py39"] } + +[features] +python = ["dep:pyo3"] +``` + +```rust +// ... an entire crate of pure Rust, then at the very bottom: + +#[cfg(feature = "python")] +mod python { + // the ONLY module allowed to say `use pyo3` +} +``` + +Why this is load-bearing: + +- **crates.io users pay nothing.** No pyo3 in their dependency tree, no + Python toolchain at build time, no compile-time cost for a face they + never see. +- **Tests need no Python.** `cargo test` exercises all the logic. This + repo's CI runs exactly that — plain `cargo test` on a machine with no + Python build setup — and this chapter's tests pass there. The + constraint isn't a workaround; it *is* the lesson. +- **The boundary stays honest.** If binding code can only live in one + gated module, business logic can't quietly migrate into it. + +`abi3-py39` is part of the rule: one compiled wheel works on every +CPython ≥ 3.9, instead of one wheel per Python version. + +## Rule 2: A Pythonic Face on a Rusty Core + +Python users should never sense they're holding Rust. Design the surface +they touch to match what a Python author would have written: + +```rust +#[pymethods] +impl PyCatalog { + #[pyo3(signature = (name, body = ""))] // keyword args + defaults + fn add(&mut self, name: &str, body: &str) -> PyResult<()> { ... } + + #[getter] // catalog.names, a property + fn names(&self) -> Vec { ... } + + fn __len__(&self) -> usize { ... } // len(catalog) + fn __contains__(&self, name: &str) -> bool { ... } // "x" in catalog + fn __repr__(&self) -> String { ... } // sensible repr() +} +``` + +```python +catalog = Catalog() +catalog.add("greeting", body="hello, world") +len(catalog) # 1 +"greeting" in catalog # True +repr(catalog) # 'Catalog(entries=1)' +``` + +Meanwhile the Rust API stays Rusty: borrowing getters, `Option` for +optional lookups, `Result` for failures, iterators instead of collected +lists. The two faces are *both* idiomatic because the boundary translates +between them — neither API contorts to imitate the other. + +One trick worth stealing: implement `Display` on the core type and make +`__repr__` call `to_string()`. The repr is written once, in the core, +and can never drift between ecosystems. + +## Rule 3: Errors Map at the Boundary — Once + +The core defines **one error enum**. Every fallible function returns it. +At the edge, **one** `From` impl converts it to typed Python exceptions: + +```rust +// CORE: errors as data, no Python anywhere +pub enum CatalogError { + NotFound(String), // a failed lookup + DuplicateName(String), // bad input + Truncated { offset: usize }, // bad data + InvalidUtf8 { offset: usize }, +} + +// BOUNDARY: the single mapping layer +impl From for PyErr { + fn from(err: CatalogError) -> PyErr { + let message = err.to_string(); + match exception_kind(&err) { + ExceptionKind::KeyError => PyKeyError::new_err(message), + ExceptionKind::ValueError => PyValueError::new_err(message), + } + } +} +``` + +Now every binding is a one-liner — `self.inner.require(name)?` — and a +missing key raises a real `KeyError` in Python, exactly as a dict would. + +The discipline that makes this durable: + +- **No catch-all arm.** The mapping `match` lists every variant. Add a + new error variant and the compiler stops you at the mapping and asks + "and what exception is that?" +- **No stringly-typed errors cross the boundary.** A `Result` + can only ever become one generic exception; an enum can become + `KeyError` here and `ValueError` there. Map *types*, not message text. +- **No hand-built `PyErr` in bindings.** If a binding constructs its own + exception, the policy has started to scatter. All roads go through the + one `From` impl. + +## Rule 4: Know What Crosses the Line + +The zero-copy mindset is about the FFI cost model. The rules are +asymmetric: + +| Direction | Signature | Cost | +|-----------|-----------|------| +| Python `bytes` → Rust | `&[u8]` | **Free** — PyO3 borrows the caller's buffer | +| Python `str` → Rust | `&str` | **Free** — borrowed for the call | +| Rust → Python `str` | `String` / `.to_owned()` | **A copy** — Python strings own their memory | +| Rust → Python `bytes` | `PyBytes::new(py, ...)` | **A copy** — same reason | + +So the house style is: + +- **Keep the core zero-copy.** Functions like + `fn peek_names(buf: &[u8]) -> Result, _>` return *views* into + the input — no allocation, and the lifetime ties the views to the + buffer at compile time. Rust callers get the full benefit. +- **Accept borrows at the boundary.** Taking `&[u8]` instead of + `Vec` means Python `bytes` comes in without a copy. +- **Pay the copy only on the way out**, where it's unavoidable — the + `.map(str::to_owned)` in a binding is an honest, visible cost, not an + accident. +- **Don't validate what you skip.** The chapter's `peek_names` never + UTF-8-checks the bodies it steps over. Zero-copy thinking is really + zero-*waste* thinking. + +For large binary payloads, Python's buffer protocol (`memoryview`) can +avoid even the outbound copy, at the price of pinning Rust memory while +Python holds the view. Know it exists; reach for it when profiling says +the copy matters, not before. + +## Rule 5: One Source of Truth + +The crates.io crate and the PyPI wheel are the *same code*, so keep them +provably the same: + +- **One repo, one version number.** The wheel's version is the crate's + version (maturin reads it straight from `Cargo.toml`). Never let the + two surfaces drift apart. +- **Behavior parity by construction.** Every binding delegates to the + core; the core's tests are therefore testing both ecosystems' behavior. + If a binding contains logic, parity is on the honor system — don't. +- **README examples for both audiences.** A user landing from crates.io + and a user landing from PyPI should each find a usage example in their + own language: + +```rust +// Rust users +let mut catalog = Catalog::new(); +catalog.add("greeting", "hello, world")?; +assert_eq!(catalog.require("greeting")?, "hello, world"); +``` + +```python +# Python users +from ch07_pyo3_house_style import Catalog +catalog = Catalog() +catalog.add("greeting", body="hello, world") +assert catalog.require("greeting") == "hello, world" +``` + +## Summary + +| House rule | Mechanism | What it buys | +|-----------|-----------|--------------| +| Core never mentions Python | optional `pyo3` + `#[cfg(feature = "python")]` | Rust users pay nothing; tests need no Python | +| Pythonic face, Rusty core | `#[pyo3(signature)]`, `#[getter]`, dunders | Both APIs idiomatic, neither contorted | +| One error enum, one mapping | `From for PyErr`, no catch-alls | Typed exceptions; compiler-enforced coverage | +| Know what crosses the line | borrow in (`&[u8]`), copy out (`to_owned`) | Zero-copy core; honest, visible boundary costs | +| One source of truth | delegation-only bindings, shared version | crates.io and PyPI can't drift apart | + +## Try It + +```bash +cargo test -p ch07-pyo3-house-style # the core — no Python required +cargo check -p ch07-pyo3-house-style --features python # compile the boundary layer too +``` + +## Next Steps + +Open `src/lib.rs` to see the whole style in one annotated file — the +feature-gated `python` module at the bottom is the entire binding layer. +Then work the exercises in `exercises/`, which practice the same +disciplines (error design, exception mapping, borrow-don't-clone, API +shape) in pure Rust. diff --git a/ch07-pyo3-house-style/exercises/Cargo.toml b/ch07-pyo3-house-style/exercises/Cargo.toml new file mode 100644 index 0000000..c18528a --- /dev/null +++ b/ch07-pyo3-house-style/exercises/Cargo.toml @@ -0,0 +1,11 @@ +# Note: NO pyo3 here, and that's deliberate. The house-style disciplines — +# designed error enums, exhaustive exception mapping, borrow-don't-clone, +# Pythonic API shape — are all pure-Rust design problems. You practice them +# with nothing but the compiler. + +[package] +name = "ch07-exercises" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Exercises for Chapter 7: PyO3 House Style" diff --git a/ch07-pyo3-house-style/exercises/src/lib.rs b/ch07-pyo3-house-style/exercises/src/lib.rs new file mode 100644 index 0000000..976870b --- /dev/null +++ b/ch07-pyo3-house-style/exercises/src/lib.rs @@ -0,0 +1,412 @@ +//! # Chapter 7 Exercises: PyO3 House Style +//! +//! These exercises build the boundary disciplines for a small settings +//! store that will one day grow a Python face. Notice what's NOT here: +//! pyo3. Every house-style decision — the error enum, the exception +//! mapping, the zero-copy signatures, the Pythonic surface — is a +//! pure-Rust design problem, and the exercises build on each other. +//! +//! Run tests: `cargo test -p ch07-exercises` + +// These allows are intentional: exercise stubs have unused parameters +// and fields until the student fills in the todo!() markers. +#![allow(unused_variables, dead_code)] + +use std::fmt; + +// ============================================================ +// Exercise 1: From Stringly-Typed to a Designed Error Enum +// ============================================================ +// +// Python version — typed exceptions with useful messages: +// ```python +// class SettingsError(Exception): ... +// +// class KeyMissing(SettingsError): +// def __init__(self, key): +// super().__init__(f"no such key: {key}") +// +// class KeyExists(SettingsError): +// def __init__(self, key): +// super().__init__(f"key already set: {key}") +// +// class InvalidValue(SettingsError): +// def __init__(self, key, reason): +// super().__init__(f"invalid value for {key}: {reason}") +// ``` +// +// The enum is defined for you — your job is the Display impl, which is +// the message every user (Rust OR Python) will eventually read. +// Match the formats exactly: +// +// KeyMissing("retries") -> "no such key: retries" +// KeyExists("retries") -> "key already set: retries" +// InvalidValue { key: "retries", +// reason: "empty value" } -> "invalid value for retries: empty value" + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SettingsError { + KeyMissing(String), + KeyExists(String), + InvalidValue { key: String, reason: String }, +} + +impl fmt::Display for SettingsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + todo!("Match each variant and write! the message formats shown above") + } +} + +impl std::error::Error for SettingsError {} + +// ============================================================ +// Exercise 2: The Exception-Mapping Layer +// ============================================================ +// +// Python version (what the boundary will eventually do): +// ```python +// # A failed lookup is a KeyError; bad input is a ValueError. +// # That's what Python users expect from dict-like objects. +// ``` +// +// Write the single function that decides which Python exception each +// error becomes: +// +// KeyMissing -> KeyError (a failed lookup) +// KeyExists -> ValueError (bad input) +// InvalidValue -> ValueError (bad input) +// +// House style: NO catch-all `_ =>` arm. Match every variant explicitly, +// so adding a new error variant later forces a conscious mapping +// decision instead of silently defaulting. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExceptionKind { + KeyError, + ValueError, +} + +pub fn exception_kind(err: &SettingsError) -> ExceptionKind { + todo!("Exhaustively match err and return the right ExceptionKind") +} + +// ============================================================ +// Exercise 3: Borrow, Don't Clone +// ============================================================ +// +// Python version — every slice is a copy, and nobody thinks about it: +// ```python +// def first_word(s: str) -> str: +// return s.split(" ")[0] # allocates a new str +// +// def payload(frame: bytes) -> bytes | None: +// if frame[:2] == b"\xc7\x07": +// return frame[2:] # copies the rest of the buffer +// return None +// ``` +// +// In Rust, return *views* instead. `&str` and `&[u8]` are a pointer and +// a length into memory the caller already owns — no allocation, and the +// borrow checker guarantees the view can't outlive the buffer. +// +// 1. `first_word`: return the text before the first space (or the whole +// string if there's no space) — as a slice of the input. The tests +// check the POINTER, not just the contents: a `.to_string()` answer +// will have the right text and still fail. +// 2. `payload`: if `frame` starts with the magic bytes [0xC7, 0x07], +// return everything after them as a borrowed slice; otherwise None. + +pub fn first_word(s: &str) -> &str { + todo!("Return a slice of s — try s.find(' ') or s.split(' ').next()") +} + +pub fn payload(frame: &[u8]) -> Option<&[u8]> { + todo!("Check for the [0xC7, 0x07] prefix, then slice past it") +} + +// ============================================================ +// Exercise 4: A Core Type with a Pythonic Surface +// ============================================================ +// +// Python version: +// ```python +// class Settings: +// """Write-once settings: keys can be set exactly once.""" +// def __init__(self): +// self._entries = {} +// +// def set(self, key, value): +// if not value: +// raise InvalidValue(key, "empty value") +// if key in self._entries: +// raise KeyExists(key) +// self._entries[key] = value +// +// def get(self, key): +// return self._entries.get(key) # None if missing +// +// def require(self, key): +// if key not in self._entries: +// raise KeyMissing(key) +// return self._entries[key] +// +// def __repr__(self): +// return f"Settings(keys={len(self._entries)})" +// ``` +// +// Implement `set`, `get`, `require`, and Display. Use the SettingsError +// variants from Exercise 1 — note that `get` returns Option (the Rust +// face) while `require` returns Result (the shape the boundary layer +// will turn into a KeyError). A core designed for two ecosystems offers +// both. + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Settings { + entries: Vec<(String, String)>, +} + +impl Settings { + pub fn new() -> Self { + Self::default() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Set a key for the first time. + /// + /// - Empty `value` -> InvalidValue with reason "empty value" + /// - Key already present -> KeyExists + pub fn set(&mut self, key: &str, value: &str) -> Result<(), SettingsError> { + todo!("Validate value, check for the key, then push (key, value)") + } + + /// Look up a value, Rust-style: Option, borrowed. + pub fn get(&self, key: &str) -> Option<&str> { + todo!("Find the entry and return the value as &str") + } + + /// Look up a value, boundary-style: a missing key is an error. + pub fn require(&self, key: &str) -> Result<&str, SettingsError> { + todo!("Build on get() — map None to KeyMissing") + } +} + +/// Display doubles as Python's __repr__: "Settings(keys=N)" +impl fmt::Display for Settings { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + todo!("Write Settings(keys=N) using self.len()") + } +} + +// ============================================================ +// Exercise 5: The Boundary Function +// ============================================================ +// +// In the real binding layer, `impl From for PyErr` turns +// every core error into a typed Python exception, and each binding is a +// one-line delegation. Model that here without pyo3: a "boundary error" +// is the pair (which exception, what message). +// +// What the real thing looks like (don't write this — read it): +// ```text +// impl From for PyErr { +// fn from(err: SettingsError) -> PyErr { +// let message = err.to_string(); +// match exception_kind(&err) { +// ExceptionKind::KeyError => PyKeyError::new_err(message), +// ExceptionKind::ValueError => PyValueError::new_err(message), +// } +// } +// } +// ``` +// +// 1. `to_boundary_error`: compose Exercise 1 (the message) with +// Exercise 2 (the kind). +// 2. `boundary_require`: delegate to Settings::require, map the error, +// and return an OWNED String — the borrowed &str must be copied to +// cross the boundary, because a Python str owns its memory. + +pub fn to_boundary_error(err: SettingsError) -> (ExceptionKind, String) { + todo!("Pair exception_kind(&err) with err.to_string()") +} + +pub fn boundary_require(settings: &Settings, key: &str) -> Result { + todo!("require + map_err(to_boundary_error) + an owned copy of the value") +} + +// ============================================================ +// Tests — do not modify below this line +// ============================================================ + +#[cfg(test)] +mod tests { + use super::*; + + // Exercise 1 + #[test] + fn ex1_key_missing_message() { + let err = SettingsError::KeyMissing("retries".to_string()); + assert_eq!(err.to_string(), "no such key: retries"); + } + + #[test] + fn ex1_key_exists_message() { + let err = SettingsError::KeyExists("retries".to_string()); + assert_eq!(err.to_string(), "key already set: retries"); + } + + #[test] + fn ex1_invalid_value_message() { + let err = SettingsError::InvalidValue { + key: "retries".to_string(), + reason: "empty value".to_string(), + }; + assert_eq!(err.to_string(), "invalid value for retries: empty value"); + } + + // Exercise 2 + #[test] + fn ex2_missing_key_is_key_error() { + let err = SettingsError::KeyMissing("x".to_string()); + assert_eq!(exception_kind(&err), ExceptionKind::KeyError); + } + + #[test] + fn ex2_bad_input_is_value_error() { + assert_eq!( + exception_kind(&SettingsError::KeyExists("x".to_string())), + ExceptionKind::ValueError + ); + assert_eq!( + exception_kind(&SettingsError::InvalidValue { + key: "x".to_string(), + reason: "empty value".to_string(), + }), + ExceptionKind::ValueError + ); + } + + // Exercise 3 + #[test] + fn ex3_first_word_before_space() { + assert_eq!(first_word("hello world"), "hello"); + } + + #[test] + fn ex3_first_word_whole_string() { + assert_eq!(first_word("single"), "single"); + } + + #[test] + fn ex3_first_word_is_zero_copy() { + let input = "borrow me"; + // Same pointer as the input — proof it's a view, not a copy. + assert_eq!(first_word(input).as_ptr(), input.as_ptr()); + } + + #[test] + fn ex3_payload_with_magic() { + let frame = [0xC7, 0x07, 1, 2, 3]; + assert_eq!(payload(&frame), Some(&frame[2..])); + } + + #[test] + fn ex3_payload_is_zero_copy() { + let frame = [0xC7, 0x07, 9, 9]; + let p = payload(&frame).unwrap(); + assert_eq!(p.as_ptr(), frame[2..].as_ptr()); + } + + #[test] + fn ex3_payload_without_magic() { + assert_eq!(payload(&[0x00, 0x07, 1]), None); + assert_eq!(payload(&[0xC7]), None); + assert_eq!(payload(&[]), None); + } + + // Exercise 4 + #[test] + fn ex4_set_and_get() { + let mut settings = Settings::new(); + settings.set("retries", "3").unwrap(); + assert_eq!(settings.get("retries"), Some("3")); + assert_eq!(settings.get("missing"), None); + } + + #[test] + fn ex4_set_rejects_empty_value() { + let mut settings = Settings::new(); + assert_eq!( + settings.set("retries", ""), + Err(SettingsError::InvalidValue { + key: "retries".to_string(), + reason: "empty value".to_string(), + }) + ); + assert!(settings.is_empty()); + } + + #[test] + fn ex4_set_rejects_duplicate_key() { + let mut settings = Settings::new(); + settings.set("retries", "3").unwrap(); + assert_eq!( + settings.set("retries", "5"), + Err(SettingsError::KeyExists("retries".to_string())) + ); + // The original value is untouched. + assert_eq!(settings.get("retries"), Some("3")); + } + + #[test] + fn ex4_require_found_and_missing() { + let mut settings = Settings::new(); + settings.set("retries", "3").unwrap(); + assert_eq!(settings.require("retries"), Ok("3")); + assert_eq!( + settings.require("missing"), + Err(SettingsError::KeyMissing("missing".to_string())) + ); + } + + #[test] + fn ex4_display_is_the_repr() { + let mut settings = Settings::new(); + settings.set("a", "1").unwrap(); + settings.set("b", "2").unwrap(); + assert_eq!(settings.to_string(), "Settings(keys=2)"); + } + + // Exercise 5 + #[test] + fn ex5_boundary_error_pairs_kind_and_message() { + let err = SettingsError::KeyMissing("retries".to_string()); + assert_eq!( + to_boundary_error(err), + (ExceptionKind::KeyError, "no such key: retries".to_string()) + ); + } + + #[test] + fn ex5_boundary_require_success_is_owned() { + let mut settings = Settings::new(); + settings.set("retries", "3").unwrap(); + let value: String = boundary_require(&settings, "retries").unwrap(); + assert_eq!(value, "3"); + } + + #[test] + fn ex5_boundary_require_missing_is_key_error() { + let settings = Settings::new(); + assert_eq!( + boundary_require(&settings, "missing"), + Err((ExceptionKind::KeyError, "no such key: missing".to_string())) + ); + } +} diff --git a/ch07-pyo3-house-style/src/lib.rs b/ch07-pyo3-house-style/src/lib.rs new file mode 100644 index 0000000..a21ff22 --- /dev/null +++ b/ch07-pyo3-house-style/src/lib.rs @@ -0,0 +1,714 @@ +//! # Chapter 7: PyO3 House Style +//! +//! Chapter 6 introduced the two-layer pattern: a pure-Rust core plus a thin +//! Python boundary. This chapter turns that pattern into a *house style* — +//! the conventions that let one codebase serve two ecosystems: a Rust crate +//! on crates.io and a Python wheel on PyPI, with one source of truth. +//! +//! The style has one load-bearing rule, and this crate demonstrates it +//! structurally: **the core never mentions Python.** All PyO3 code lives +//! behind a default-off `python` feature. Run the tests right now and you +//! compile zero lines of binding code and need no Python toolchain. That +//! is not a workaround for this course's CI — it IS the lesson. A +//! well-styled hybrid crate is a first-class Rust library that grows a +//! Python face only when asked: +//! +//! ```text +//! cargo test -p ch07-pyo3-house-style # core only — what crates.io users get +//! cargo check -p ch07-pyo3-house-style --features python # core + bindings — what PyPI users get +//! ``` +//! +//! Run the tests: `cargo test -p ch07-pyo3-house-style` + +use std::fmt; + +// --------------------------------------------------------------------------- +// 1. The layering rule — core code never mentions Python +// --------------------------------------------------------------------------- +// +// Everything from here down to the `python` module at the bottom of this +// file is plain Rust. It compiles without pyo3, tests without a Python +// interpreter, and could be published to crates.io as-is. +// +// Python equivalent of the *idea* (a package whose optional accelerator +// is invisible unless installed): +// +// ```python +// # pip install mylib -> pure-Python behavior +// # pip install mylib[fast] -> same API, native accelerator engaged +// try: +// from mylib._native import decode +// except ImportError: +// from mylib._pure import decode +// ``` +// +// In Rust the switch happens at compile time instead of import time: +// +// ```toml +// [dependencies] +// pyo3 = { version = "0.23", optional = true, features = ["extension-module", "abi3-py39"] } +// +// [features] +// python = ["dep:pyo3"] +// ``` +// +// `cargo build` ignores pyo3 entirely. `cargo build --features python` +// compiles the boundary layer. Rust users never pay for the Python face. + +// --------------------------------------------------------------------------- +// 2. One error enum in the core +// --------------------------------------------------------------------------- + +/// Every way this library can fail, as data. +/// +/// Python equivalent — a small exception hierarchy: +/// ```python +/// class CatalogError(Exception): ... +/// class NotFoundError(CatalogError, KeyError): ... +/// class DuplicateNameError(CatalogError, ValueError): ... +/// class CorruptDataError(CatalogError, ValueError): ... +/// ``` +/// +/// House style: the core defines ONE error enum and every fallible +/// function returns it. No `Result`, no `panic!` for expected +/// failures. Stringly-typed errors can't be matched on, can't be mapped +/// to distinct Python exceptions, and can't be extended without grepping +/// for message text. An enum can. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CatalogError { + /// A lookup for a name that isn't in the catalog. + NotFound(String), + /// An attempt to add a name that's already taken. + DuplicateName(String), + /// Encoded data ended mid-record. `offset` is where parsing stopped. + Truncated { offset: usize }, + /// Encoded data contained invalid UTF-8 at `offset`. + InvalidUtf8 { offset: usize }, +} + +impl fmt::Display for CatalogError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotFound(name) => write!(f, "no entry named {name:?}"), + Self::DuplicateName(name) => write!(f, "entry {name:?} already exists"), + Self::Truncated { offset } => write!(f, "data truncated at byte {offset}"), + Self::InvalidUtf8 { offset } => write!(f, "invalid UTF-8 at byte {offset}"), + } + } +} + +impl std::error::Error for CatalogError {} + +// --------------------------------------------------------------------------- +// 3. The mapping layer — one place where errors become exceptions +// --------------------------------------------------------------------------- + +/// Which Python exception type an error should surface as. +/// +/// This enum is the *pure-Rust model* of the boundary decision. The actual +/// `From for PyErr` impl (bottom of this file, feature-gated) +/// consumes it — but the decision itself is core logic, so it lives here +/// where `cargo test` can reach it without Python. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExceptionKind { + /// Maps to Python's `KeyError` — a failed lookup. + KeyError, + /// Maps to Python's `ValueError` — bad input or bad data. + ValueError, +} + +/// Decide, exhaustively, which Python exception each error becomes. +/// +/// Python users expect `KeyError` from a failed lookup and `ValueError` +/// from bad data — matching dict and bytes.decode() behavior. Meeting +/// that expectation is part of designing a Pythonic face. +/// +/// House style: this `match` has NO catch-all arm. When you add a new +/// error variant, the compiler stops you right here and asks "and what +/// exception is that?" — the mapping can never silently fall behind the +/// error enum. A `_ => ValueError` arm would trade that guarantee away. +pub fn exception_kind(err: &CatalogError) -> ExceptionKind { + match err { + CatalogError::NotFound(_) => ExceptionKind::KeyError, + CatalogError::DuplicateName(_) + | CatalogError::Truncated { .. } + | CatalogError::InvalidUtf8 { .. } => ExceptionKind::ValueError, + } +} + +// --------------------------------------------------------------------------- +// 4. A core type designed for two faces +// --------------------------------------------------------------------------- + +/// An ordered collection of named text snippets. +/// +/// Python users will see this as a class: +/// ```python +/// catalog = Catalog() +/// catalog.add("greeting", "hello, world") +/// catalog.require("greeting") # -> "hello, world", or KeyError +/// len(catalog) # __len__ +/// "greeting" in catalog # __contains__ +/// repr(catalog) # "Catalog(entries=1)" +/// ``` +/// +/// Rust users see an ordinary struct with borrowing getters and +/// `Result`-returning methods. Same behavior, two idiomatic faces — +/// that's the point. Neither API is a second-class translation of +/// the other. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Catalog { + /// `Vec` rather than `HashMap`: insertion order is part of the + /// contract (and of the encoded byte format below). + entries: Vec<(String, String)>, +} + +impl Catalog { + pub fn new() -> Self { + Self::default() + } + + /// Add a named snippet. Names are write-once. + /// + /// Returns `DuplicateName` instead of silently overwriting — the + /// boundary layer will surface that as `ValueError`. + pub fn add(&mut self, name: &str, body: &str) -> Result<(), CatalogError> { + if self.contains(name) { + return Err(CatalogError::DuplicateName(name.to_string())); + } + self.entries.push((name.to_string(), body.to_string())); + Ok(()) + } + + /// Look up a snippet, Rust-style: `Option`, borrowed, zero-copy. + pub fn get(&self, name: &str) -> Option<&str> { + self.entries + .iter() + .find(|(n, _)| n == name) + .map(|(_, body)| body.as_str()) + } + + /// Look up a snippet, boundary-style: a missing name is an *error*. + /// + /// Python has no `Option` — its idiom for a failed lookup is + /// `KeyError`. The core provides both shapes so the boundary layer + /// can delegate instead of re-implementing the policy. + pub fn require(&self, name: &str) -> Result<&str, CatalogError> { + self.get(name) + .ok_or_else(|| CatalogError::NotFound(name.to_string())) + } + + pub fn contains(&self, name: &str) -> bool { + self.entries.iter().any(|(n, _)| n == name) + } + + /// Iterate names in insertion order, borrowed. + pub fn names(&self) -> impl Iterator { + self.entries.iter().map(|(n, _)| n.as_str()) + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +/// `Display` doubles as the model for Python's `__repr__`. +/// +/// House style: write the repr once, in the core, and have the boundary's +/// `__repr__` call `to_string()`. One format, two ecosystems. +impl fmt::Display for Catalog { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Catalog(entries={})", self.entries.len()) + } +} + +// --------------------------------------------------------------------------- +// 5. Zero-copy parsing — what actually crosses the line +// --------------------------------------------------------------------------- +// +// The wire format: each entry is two length-prefixed chunks, +// +// [name_len: u32 LE][name bytes][body_len: u32 LE][body bytes] +// +// repeated until the buffer ends. Parsing it is where the zero-copy +// mindset shows up. In Python, slicing bytes copies: +// +// ```python +// name = buf[4:4 + name_len].decode() # allocates a new str +// ``` +// +// In Rust, `&buf[4..4 + name_len]` is a *view* — a pointer and a length +// into memory you already own. The lifetime in `fn(&[u8]) -> &str` is the +// compiler-enforced promise that the view never outlives the buffer. +// +// At the FFI line the rules are asymmetric, and worth memorizing: +// +// - Python `bytes` -> Rust `&[u8]`: FREE. PyO3 borrows the existing +// buffer; no copy on the way in. +// - Rust `&str` -> Python `str`: A COPY. Python strings own their +// memory, so borrowed data must be materialized on the way out. +// +// So the house style is: keep the core zero-copy (Rust callers get the +// full benefit), accept borrows at the boundary (free), and pay the copy +// only on returned values — the one place it's unavoidable. (For large +// binary payloads, Python's buffer protocol / `memoryview` can avoid even +// that, at the cost of pinning Rust memory; know it exists, reach for it +// only when profiling says so.) + +/// Find the byte range of the next length-prefixed chunk. +/// +/// Bounds problems become `Truncated { offset }` — pointing at the chunk +/// that failed, because "your data is cut off at byte 17" beats "index +/// out of range". +fn chunk_range(buf: &[u8], offset: usize) -> Result, CatalogError> { + let len_end = offset + 4; + if buf.len() < len_end { + return Err(CatalogError::Truncated { offset }); + } + let len = u32::from_le_bytes(buf[offset..len_end].try_into().expect("4 bytes")) as usize; + let data_end = len_end + len; + if buf.len() < data_end { + return Err(CatalogError::Truncated { offset }); + } + Ok(len_end..data_end) +} + +/// Read the next chunk as UTF-8 text — without copying. +/// +/// Note the signature: the returned `&str` borrows from `buf`. No +/// allocation happens here; we validate in place and hand back a view. +fn read_str_chunk<'a>(buf: &'a [u8], offset: &mut usize) -> Result<&'a str, CatalogError> { + let range = chunk_range(buf, *offset)?; + let text = std::str::from_utf8(&buf[range.clone()]).map_err(|_| CatalogError::InvalidUtf8 { + offset: range.start, + })?; + *offset = range.end; + Ok(text) +} + +/// Skip a chunk without reading it — bounds are checked, bytes are not. +/// +/// "Don't pay for what you don't use": skipped chunks aren't UTF-8 +/// validated, because nobody is going to look at them. +fn skip_chunk(buf: &[u8], offset: &mut usize) -> Result<(), CatalogError> { + *offset = chunk_range(buf, *offset)?.end; + Ok(()) +} + +impl Catalog { + /// Encode the catalog to bytes. The one place encoding allocates. + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::new(); + for (name, body) in &self.entries { + for field in [name, body] { + out.extend_from_slice(&(field.len() as u32).to_le_bytes()); + out.extend_from_slice(field.as_bytes()); + } + } + out + } + + /// Decode a catalog from bytes, validating everything. + /// + /// The names and bodies are *stored*, so this copies them out of the + /// buffer — an honest copy, because the `Catalog` outlives `buf`. + /// Compare with `peek_names` below, which doesn't. + pub fn from_bytes(buf: &[u8]) -> Result { + let mut catalog = Catalog::new(); + let mut offset = 0; + while offset < buf.len() { + let name = read_str_chunk(buf, &mut offset)?; + let body = read_str_chunk(buf, &mut offset)?; + catalog.add(name, body)?; + } + Ok(catalog) + } +} + +/// List the entry names in an encoded catalog — zero-copy, zero-waste. +/// +/// Python equivalent (which copies every slice it touches): +/// ```python +/// def peek_names(buf: bytes) -> list[str]: +/// names, offset = [], 0 +/// while offset < len(buf): +/// n = int.from_bytes(buf[offset:offset + 4], "little") +/// names.append(buf[offset + 4:offset + 4 + n].decode()) +/// offset += 4 + n +/// n = int.from_bytes(buf[offset:offset + 4], "little") +/// offset += 4 + n # skip the body +/// return names +/// ``` +/// +/// The Rust version allocates nothing for the names (they're views into +/// `buf`) and never even UTF-8-validates the bodies it skips. The +/// returned strings are borrowed — the signature ties their lifetime to +/// the buffer's, and the compiler enforces it. +pub fn peek_names(buf: &[u8]) -> Result, CatalogError> { + let mut names = Vec::new(); + let mut offset = 0; + while offset < buf.len() { + names.push(read_str_chunk(buf, &mut offset)?); + skip_chunk(buf, &mut offset)?; + } + Ok(names) +} + +// --------------------------------------------------------------------------- +// 6. The boundary layer — feature-gated, thin, and Pythonic +// --------------------------------------------------------------------------- +// +// Everything below compiles ONLY with `--features python`. This module is +// the entire Python face of the crate: it converts types, maps errors, +// and adds dunder methods. It contains no business logic — every method +// is a delegation to the core above. If you find yourself writing an +// `if` in this module that isn't about conversion, it belongs in the core. +// +// One source of truth, two ecosystems served: +// +// ```python +// from ch07_pyo3_house_style import Catalog, peek_names +// +// catalog = Catalog() +// catalog.add("greeting", body="hello, world") # kwargs via #[pyo3(signature)] +// catalog.require("greeting") # "hello, world" +// catalog.require("missing") # raises KeyError +// catalog.add("greeting", body="again") # raises ValueError +// peek_names(catalog.to_bytes()) # ["greeting"] +// ``` + +#[cfg(feature = "python")] +mod python { + use pyo3::exceptions::{PyKeyError, PyValueError}; + use pyo3::prelude::*; + use pyo3::types::PyBytes; + + use super::{exception_kind, Catalog, CatalogError, ExceptionKind}; + + /// THE error-mapping layer. Singular. + /// + /// Because this impl exists, every binding below can use `?` on a + /// core `Result` and the right typed Python exception comes out. + /// No binding ever constructs a PyErr by hand — if one did, the + /// mapping policy would start to scatter. + impl From for PyErr { + fn from(err: CatalogError) -> PyErr { + let message = err.to_string(); + match exception_kind(&err) { + ExceptionKind::KeyError => PyKeyError::new_err(message), + ExceptionKind::ValueError => PyValueError::new_err(message), + } + } + } + + /// The Python class. A newtype over the core — nothing more. + #[pyclass(name = "Catalog")] + struct PyCatalog { + inner: Catalog, + } + + #[pymethods] + impl PyCatalog { + #[new] + fn new() -> Self { + Self { + inner: Catalog::new(), + } + } + + /// `#[pyo3(signature = ...)]` gives Python callers keyword + /// arguments and defaults — `catalog.add("name", body="...")` — + /// without contorting the Rust API to match. + #[pyo3(signature = (name, body = ""))] + fn add(&mut self, name: &str, body: &str) -> PyResult<()> { + // `?` + the From impl above = the entire error story. + self.inner.add(name, body)?; + Ok(()) + } + + /// `Option` surfaces as `str | None` — Pythonic for an + /// optional lookup. The `.map(str::to_owned)` is the honest copy + /// at the boundary: a Python str must own its memory. + fn get(&self, name: &str) -> Option { + self.inner.get(name).map(str::to_owned) + } + + /// The error-raising lookup: `PyResult` + the From impl turn + /// `CatalogError::NotFound` into a real `KeyError`. + fn require(&self, name: &str) -> PyResult { + Ok(self.inner.require(name)?.to_owned()) + } + + /// A `#[getter]` makes this `catalog.names` (a property), not + /// `catalog.names()` — match what a Python author would write. + #[getter] + fn names(&self) -> Vec { + self.inner.names().map(str::to_owned).collect() + } + + /// Returning `PyBytes` explicitly: one allocation, owned by + /// Python, no intermediate `Vec` handoff semantics to explain. + fn to_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { + PyBytes::new(py, &self.inner.to_bytes()) + } + + /// `&[u8]` here means Python `bytes` comes in WITHOUT a copy — + /// PyO3 borrows the caller's buffer for the duration of the call. + #[staticmethod] + fn from_bytes(data: &[u8]) -> PyResult { + Ok(Self { + inner: Catalog::from_bytes(data)?, + }) + } + + // The dunder protocol: len(), `in`, and repr() — implemented by + // delegation, so Python ergonomics never fork from core behavior. + + fn __len__(&self) -> usize { + self.inner.len() + } + + fn __contains__(&self, name: &str) -> bool { + self.inner.contains(name) + } + + fn __repr__(&self) -> String { + // Display IS the repr. Written once, in the core. + self.inner.to_string() + } + } + + /// Free function binding. The core returns borrowed `Vec<&str>`; + /// the boundary materializes owned strings because they're about to + /// become Python objects. Borrow inside, copy at the line. + #[pyfunction] + fn peek_names(buf: &[u8]) -> PyResult> { + Ok(super::peek_names(buf)? + .into_iter() + .map(str::to_owned) + .collect()) + } + + /// The module definition — the table of contents of the Python face. + /// (A shipping project would build this into a wheel with maturin.) + #[pymodule] + fn ch07_pyo3_house_style(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_function(wrap_pyfunction!(peek_names, m)?)?; + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Tests — note: pure Rust, no Python anywhere +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // The error enum + + #[test] + fn errors_display_with_context() { + assert_eq!( + CatalogError::NotFound("greeting".to_string()).to_string(), + "no entry named \"greeting\"" + ); + assert_eq!( + CatalogError::Truncated { offset: 17 }.to_string(), + "data truncated at byte 17" + ); + } + + // The mapping layer — every variant has a deliberate destination + + #[test] + fn failed_lookup_maps_to_key_error() { + let err = CatalogError::NotFound("x".to_string()); + assert_eq!(exception_kind(&err), ExceptionKind::KeyError); + } + + #[test] + fn bad_input_maps_to_value_error() { + assert_eq!( + exception_kind(&CatalogError::DuplicateName("x".to_string())), + ExceptionKind::ValueError + ); + assert_eq!( + exception_kind(&CatalogError::Truncated { offset: 0 }), + ExceptionKind::ValueError + ); + assert_eq!( + exception_kind(&CatalogError::InvalidUtf8 { offset: 4 }), + ExceptionKind::ValueError + ); + } + + // The core type + + #[test] + fn add_and_get() { + let mut catalog = Catalog::new(); + catalog.add("greeting", "hello, world").unwrap(); + assert_eq!(catalog.get("greeting"), Some("hello, world")); + assert_eq!(catalog.get("missing"), None); + } + + #[test] + fn require_found_and_missing() { + let mut catalog = Catalog::new(); + catalog.add("greeting", "hello").unwrap(); + assert_eq!(catalog.require("greeting"), Ok("hello")); + assert_eq!( + catalog.require("missing"), + Err(CatalogError::NotFound("missing".to_string())) + ); + } + + #[test] + fn duplicate_names_rejected() { + let mut catalog = Catalog::new(); + catalog.add("greeting", "hello").unwrap(); + assert_eq!( + catalog.add("greeting", "again"), + Err(CatalogError::DuplicateName("greeting".to_string())) + ); + // The original entry is untouched. + assert_eq!(catalog.get("greeting"), Some("hello")); + } + + #[test] + fn names_preserve_insertion_order() { + let mut catalog = Catalog::new(); + catalog.add("b", "2").unwrap(); + catalog.add("a", "1").unwrap(); + let names: Vec<&str> = catalog.names().collect(); + assert_eq!(names, vec!["b", "a"]); + } + + #[test] + fn len_contains_empty() { + let mut catalog = Catalog::new(); + assert!(catalog.is_empty()); + catalog.add("x", "1").unwrap(); + assert_eq!(catalog.len(), 1); + assert!(catalog.contains("x")); + assert!(!catalog.contains("y")); + } + + #[test] + fn display_is_the_repr() { + let mut catalog = Catalog::new(); + catalog.add("a", "1").unwrap(); + catalog.add("b", "2").unwrap(); + assert_eq!(catalog.to_string(), "Catalog(entries=2)"); + } + + // The wire format + + #[test] + fn bytes_roundtrip() { + let mut catalog = Catalog::new(); + catalog.add("greeting", "hello, world").unwrap(); + catalog.add("farewell", "goodbye").unwrap(); + + let decoded = Catalog::from_bytes(&catalog.to_bytes()).unwrap(); + assert_eq!(decoded, catalog); + } + + #[test] + fn empty_catalog_roundtrip() { + let catalog = Catalog::new(); + let bytes = catalog.to_bytes(); + assert!(bytes.is_empty()); + assert_eq!(Catalog::from_bytes(&bytes).unwrap(), catalog); + } + + #[test] + fn truncated_length_prefix() { + // Two bytes can't hold a 4-byte length prefix. + let err = Catalog::from_bytes(&[1, 0]).unwrap_err(); + assert_eq!(err, CatalogError::Truncated { offset: 0 }); + } + + #[test] + fn truncated_chunk_body() { + // Length says 10 bytes follow; only 1 does. + let err = Catalog::from_bytes(&[10, 0, 0, 0, b'a']).unwrap_err(); + assert_eq!(err, CatalogError::Truncated { offset: 0 }); + } + + #[test] + fn invalid_utf8_reports_offset() { + // A 3-byte name chunk containing invalid UTF-8. + let err = Catalog::from_bytes(&[3, 0, 0, 0, 0xFF, 0xFE, 0xFF]).unwrap_err(); + assert_eq!(err, CatalogError::InvalidUtf8 { offset: 4 }); + } + + #[test] + fn duplicate_in_encoded_data_rejected() { + let mut catalog = Catalog::new(); + catalog.add("a", "1").unwrap(); + let mut bytes = catalog.to_bytes(); + let again = bytes.clone(); + bytes.extend_from_slice(&again); // entry "a" appears twice + + assert_eq!( + Catalog::from_bytes(&bytes).unwrap_err(), + CatalogError::DuplicateName("a".to_string()) + ); + } + + // Zero-copy parsing + + #[test] + fn peek_names_lists_names_in_order() { + let mut catalog = Catalog::new(); + catalog.add("first", "1").unwrap(); + catalog.add("second", "2").unwrap(); + + let bytes = catalog.to_bytes(); + assert_eq!(peek_names(&bytes).unwrap(), vec!["first", "second"]); + } + + #[test] + fn peek_names_borrows_from_the_buffer() { + let mut catalog = Catalog::new(); + catalog.add("zero-copy", "body").unwrap(); + let bytes = catalog.to_bytes(); + + let names = peek_names(&bytes).unwrap(); + // The returned &str points INTO `bytes` — same memory, no copy. + // (The name chunk starts after its 4-byte length prefix.) + assert_eq!(names[0].as_ptr(), bytes[4..].as_ptr()); + } + + #[test] + fn peek_names_skips_body_validation() { + // name "a" is valid; the BODY is invalid UTF-8. + let bytes = [1, 0, 0, 0, b'a', 2, 0, 0, 0, 0xFF, 0xFE]; + + // Full decoding validates everything and fails... + assert_eq!( + Catalog::from_bytes(&bytes).unwrap_err(), + CatalogError::InvalidUtf8 { offset: 9 } + ); + // ...but peeking never reads the bodies it skips. + assert_eq!(peek_names(&bytes).unwrap(), vec!["a"]); + } + + #[test] + fn peek_names_still_checks_bounds() { + // Valid name, then a body whose length overruns the buffer. + let bytes = [1, 0, 0, 0, b'a', 99, 0, 0, 0, b'x']; + assert_eq!( + peek_names(&bytes).unwrap_err(), + CatalogError::Truncated { offset: 5 } + ); + } +} From 2da75ffa7e9a384f57c7a9a081dbcd87e7f61930 Mon Sep 17 00:00:00 2001 From: hartsock Date: Fri, 5 Jun 2026 20:05:48 -0400 Subject: [PATCH 4/5] Add Chapter 8: Escaping the GIL (Send/Sync, rayon, releasing the GIL) Maps Python's GIL/threading/multiprocessing model to Rust's compile-time concurrency: scoped threads, Send/Sync (with compile_fail doctests showing data races the compiler rejects), Arc>, mpsc channels, and rayon. PyO3 allow_threads example is gated behind a default-off `python` feature. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 2 + ch08-escaping-the-gil/Cargo.toml | 18 + ch08-escaping-the-gil/README.md | 203 ++++++++ ch08-escaping-the-gil/exercises/Cargo.toml | 9 + ch08-escaping-the-gil/exercises/src/lib.rs | 381 ++++++++++++++ ch08-escaping-the-gil/src/lib.rs | 580 +++++++++++++++++++++ 6 files changed, 1193 insertions(+) create mode 100644 ch08-escaping-the-gil/Cargo.toml create mode 100644 ch08-escaping-the-gil/README.md create mode 100644 ch08-escaping-the-gil/exercises/Cargo.toml create mode 100644 ch08-escaping-the-gil/exercises/src/lib.rs create mode 100644 ch08-escaping-the-gil/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 2893935..6226e6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ members = [ "ch06-ffi-pyo3/exercises", "ch07-pyo3-house-style", "ch07-pyo3-house-style/exercises", + "ch08-escaping-the-gil", + "ch08-escaping-the-gil/exercises", ] [workspace.package] diff --git a/ch08-escaping-the-gil/Cargo.toml b/ch08-escaping-the-gil/Cargo.toml new file mode 100644 index 0000000..74ca53e --- /dev/null +++ b/ch08-escaping-the-gil/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ch08-escaping-the-gil" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Chapter 8: Escaping the GIL — Send/Sync, rayon, and real threads" + +[dependencies] +rayon = "1" + +# PyO3 is optional: the `python` feature is OFF by default, so building and +# testing this chapter requires no Python installation. Enable it with +# `cargo build -p ch08-escaping-the-gil --features python` to compile the +# GIL-releasing extension example. +pyo3 = { version = "0.23", optional = true, features = ["extension-module", "abi3-py39"] } + +[features] +python = ["dep:pyo3"] diff --git a/ch08-escaping-the-gil/README.md b/ch08-escaping-the-gil/README.md new file mode 100644 index 0000000..febdc4b --- /dev/null +++ b/ch08-escaping-the-gil/README.md @@ -0,0 +1,203 @@ +# Chapter 8: Escaping the GIL + +## The Big Idea + +Every Python developer eventually hits the same wall: you have a CPU-bound +loop, you reach for `threading`, and... nothing gets faster. The Global +Interpreter Lock (GIL) lets only one thread execute Python bytecode at a +time. Threads are great for *waiting* (network, disk) and useless for +*computing*. The sanctioned workaround, `multiprocessing`, gets real +parallelism by spawning whole interpreter processes and pickling every +argument and result across process boundaries — heavyweight, copy-everything +parallelism. + +Rust has no GIL. Its threads are real OS threads that genuinely run on +multiple cores. So what stops them from corrupting shared data the way the +GIL "protects" Python objects? Two marker traits — **`Send`** (safe to move +to another thread) and **`Sync`** (safe to share between threads) — checked +at *compile time*. The type system does the GIL's job without serializing +execution: a data race in Rust is not a 3 a.m. heisenbug, it's a compile +error with a line number. + +## Python Analogies + +### `threading.Thread` = `std::thread` (but actually parallel) + +```python +import threading + +def worker(start, end, results, i): + results[i] = count_primes(start, end) # GIL: one thread at a time + +threads = [threading.Thread(target=worker, args=(...)) for i in range(4)] +for t in threads: t.start() +for t in threads: t.join() +# Elapsed time: the same as one thread. Often worse. +``` + +```rust +std::thread::scope(|s| { + let handles: Vec<_> = chunks + .map(|(start, end)| s.spawn(move || count_primes(start, end))) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).sum::() +}); +// Elapsed time: roughly divided by your core count. +``` + +**Key insight:** the API shape is nearly identical — spawn, join, collect. +What changes is the semantics: Rust threads run simultaneously. +`thread::scope` adds a guarantee Python can't express: every spawned thread +provably finishes before the scope exits, so threads may safely *borrow* +local data instead of copying it. + +### The GIL = `Send` + `Sync` (a lock at runtime vs a proof at compile time) + +Python makes shared objects "safe" by never letting two threads run at +once. Rust makes them safe by refusing to compile programs that share the +wrong types: + +```rust +use std::rc::Rc; +use std::thread; + +let data = Rc::new(vec![1, 2, 3]); // Rc: non-atomic refcount +thread::spawn(move || data.len()); // ERROR: `Rc>` cannot be + // sent between threads safely +``` + +`Rc`'s reference count is plain (non-atomic) — exactly like CPython's +refcounting, which is precisely *why* CPython needs a GIL. Swap in `Arc` +(atomic reference count) and the same code compiles. The compiler error you +just read **is a data race that never got to exist**. + +| Single-threaded type | Thread-safe sibling | Python's version | +|----------------------|--------------------|------------------| +| `Rc` | `Arc` | every object (GIL-guarded refcount) | +| `RefCell` | `Mutex` / `RwLock` | every object (GIL-guarded mutation) | + +### `threading.Lock` = `Mutex` (but the lock *owns* the data) + +```python +counter = 0 +lock = threading.Lock() + +def worker(): + global counter + with lock: + counter += 1 + # Nothing stops a careless coworker from writing + # `counter += 1` without taking the lock. Hope is the strategy. +``` + +```rust +let counter = Arc::new(Mutex::new(0)); + +// The i32 lives INSIDE the Mutex. There is no way to reach it except +// .lock(), and the lock releases when the guard goes out of scope. +*counter.lock().unwrap() += 1; +``` + +**Key insight:** Python's lock and the data it protects are unrelated +objects, associated only by convention. `Mutex` *contains* its data — +forgetting to take the lock is not a bug you can write. + +### `queue.Queue` = `mpsc::channel` + +```python +from queue import Queue +q = Queue() +# workers: q.put(result) +# main: q.get() — but how many gets? Sentinels? Poison pills? +``` + +```rust +let (tx, rx) = std::sync::mpsc::channel(); +// workers: tx.send(result) +// main: rx.iter().sum() — ends automatically when all senders drop +``` + +**Key insight:** same multi-producer queue, but channel closure is tied to +ownership: when every `Sender` is dropped, the receiver's iterator simply +ends. No sentinel values, no counting expected messages. Channels also pair +beautifully with the ownership model: a value *moves* through the channel, +so sender and receiver can never touch it at the same time. + +### `multiprocessing.Pool` = `rayon` (without the pickling tax) + +```python +from multiprocessing import Pool + +with Pool() as pool: # fork N interpreters + flags = pool.map(is_prime, range(2, N)) # pickle every argument, +total = sum(flags) # pickle every result back +``` + +```rust +use rayon::prelude::*; + +let total = (2..n).into_par_iter().filter(|&n| is_prime(n)).count(); +``` + +**Key insight:** in Python, process-based parallelism is an architectural +decision — serialization costs, shared-memory gymnastics, "can this even be +pickled?" In Rust, it's a one-word diff: `iter()` → `par_iter()`. rayon's +work-stealing pool shares the address space (zero copies) and `Send`/`Sync` +still stand guard: if the closure captured something thread-unsafe, the +one-word change would not compile. + +## The FFI Payoff: Releasing the GIL + +This is where the course arc lands. A PyO3 extension (Chapters 6–7) holds +the GIL while it touches Python objects — but pure-Rust computation doesn't +need Python at all. `py.allow_threads(...)` releases the GIL for the +duration of a closure: + +```rust +#[pyfunction] +fn count_primes_releasing_gil(py: Python<'_>, lo: u64, hi: u64) -> usize { + py.allow_threads(|| { + (lo..hi).into_par_iter().filter(|&n| is_prime(n)).count() + }) +} +``` + +Two things happen at once: + +1. **The Rust side fans out** across every core with rayon — no GIL exists + in Rust-land to stop it. +2. **The Python side keeps moving** — other Python threads run freely while + Rust computes, because the GIL is released. + +The closure can't touch Python objects — PyO3 enforces that with `Send` +bounds, the same machinery from this chapter. Python's biggest weakness +becomes a non-issue: keep writing Python, and let the hot loop escape to +threads the GIL cannot see. (This code is feature-gated behind +`--features python` in this chapter so the default build needs no Python.) + +## A Note on Testing Parallel Code + +The tests in this chapter assert one thing, many ways: **the parallel +result equals the sequential result.** They never assert on timing. +Wall-clock speedups depend on core counts, schedulers, and how noisy the +machine is — assertions like "4 threads should be 3x faster" are flaky by +construction, especially on shared CI runners. Benchmark performance in +prose and profiles; test *determinism*. + +## Summary + +| Python | Rust | What Changes | +|--------|------|-------------| +| The GIL | `Send` / `Sync` | Safety moves from a runtime lock to compile-time proof | +| `threading.Thread` | `std::thread::scope` / `spawn` | Threads actually run in parallel; scoped threads may borrow | +| Shared objects + hope | `Arc` | Atomic refcount; sharing is explicit in the type | +| `threading.Lock` + convention | `Mutex` | The lock owns the data; unlocked access won't compile | +| `queue.Queue` | `mpsc::channel` | Closes automatically when senders drop; values move | +| `multiprocessing.Pool` | `rayon` `par_iter` | No process spawn, no pickling — a one-word diff | +| C extension holding the GIL | `py.allow_threads` | Rust computes on all cores while Python keeps running | + +## Next Steps + +Open `src/lib.rs` to see these concepts in working code — including two +"data races that won't compile" captured as `compile_fail` doctests — then +try the exercises in `exercises/`. diff --git a/ch08-escaping-the-gil/exercises/Cargo.toml b/ch08-escaping-the-gil/exercises/Cargo.toml new file mode 100644 index 0000000..0706d86 --- /dev/null +++ b/ch08-escaping-the-gil/exercises/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ch08-exercises" +version = "0.1.0" +edition.workspace = true +license.workspace = true +description = "Exercises for Chapter 8: Escaping the GIL" + +[dependencies] +rayon = "1" diff --git a/ch08-escaping-the-gil/exercises/src/lib.rs b/ch08-escaping-the-gil/exercises/src/lib.rs new file mode 100644 index 0000000..3572347 --- /dev/null +++ b/ch08-escaping-the-gil/exercises/src/lib.rs @@ -0,0 +1,381 @@ +//! # Chapter 8 Exercises: Escaping the GIL +//! +//! Each exercise shows a Python snippet and asks you to write the Rust +//! equivalent — except this time, the Rust version actually runs in +//! parallel. Replace the `todo!()` markers with working code. +//! +//! Every test asserts that your parallel result equals a sequential +//! reference result. Correctness first; speed comes along for free. +//! +//! Run tests: `cargo test -p ch08-exercises` + +// These allows are intentional: exercise stubs have unused parameters +// and fields until the student fills in the todo!() markers. +#![allow(unused_variables, dead_code)] + +use std::sync::{Arc, Mutex}; + +// ============================================================ +// Exercise 1: Scoped Threads — Sum a Slice in Chunks +// ============================================================ +// +// Python version (gets NO speedup — the GIL serializes the workers): +// ```python +// import threading +// +// def sum_threaded(numbers: list[int], n_chunks: int) -> int: +// chunk_size = max(1, -(-len(numbers) // n_chunks)) # ceiling division +// chunks = [numbers[i:i + chunk_size] for i in range(0, len(numbers), chunk_size)] +// results = [0] * len(chunks) +// +// def worker(i, chunk): +// results[i] = sum(chunk) +// +// threads = [threading.Thread(target=worker, args=(i, c)) +// for i, c in enumerate(chunks)] +// for t in threads: t.start() +// for t in threads: t.join() +// return sum(results) +// ``` +// +// Write the Rust version with std::thread::scope. Because scoped +// threads are guaranteed to finish before the scope returns, they can +// BORROW the slice — no copying, no Arc needed. +// +// Hints: +// - `numbers.chunks(chunk_size)` yields borrowed sub-slices +// - `std::thread::scope(|s| { ... s.spawn(move || ...) ... })` +// - collect the handles, then `.join().unwrap()` each and sum + +pub fn sum_threaded(numbers: &[i64], n_chunks: usize) -> i64 { + todo!("Split into chunks, spawn a scoped thread per chunk, sum the partial sums") +} + +// ============================================================ +// Exercise 2: Arc> — A Shared Counter +// ============================================================ +// +// Python version: +// ```python +// import threading +// +// class SharedCounter: +// def __init__(self): +// self._value = 0 +// self._lock = threading.Lock() # lock and data: separate objects! +// +// def increment(self): +// with self._lock: +// self._value += 1 +// +// def value(self): +// with self._lock: +// return self._value +// +// def run_incrementers(counter, n_threads, increments_each): +// def worker(): +// for _ in range(increments_each): +// counter.increment() +// threads = [threading.Thread(target=worker) for _ in range(n_threads)] +// for t in threads: t.start() +// for t in threads: t.join() +// ``` +// +// In Rust the Mutex OWNS the value — there is no way to touch the u64 +// without locking. Implement `increment`, `value`, and +// `run_incrementers`. +// +// Hints: +// - `self.inner.lock().unwrap()` gives a guard; `*guard += 1` mutates +// - `run_incrementers` needs `std::thread::spawn` + a clone of the +// counter per thread (that's why SharedCounter derives Clone — cloning +// an Arc just bumps the refcount; both clones point at the same Mutex) + +#[derive(Clone)] +pub struct SharedCounter { + inner: Arc>, +} + +impl SharedCounter { + pub fn new() -> Self { + Self { + inner: Arc::new(Mutex::new(0)), + } + } + + /// Add 1 to the counter (lock, mutate, unlock). + pub fn increment(&self) { + todo!("Lock the mutex and add 1") + } + + /// Read the current value. + pub fn value(&self) -> u64 { + todo!("Lock the mutex and return a copy of the value") + } +} + +impl Default for SharedCounter { + fn default() -> Self { + Self::new() + } +} + +/// Spawn `n_threads` threads that each call `increment()` +/// `increments_each` times, and wait for all of them to finish. +pub fn run_incrementers(counter: &SharedCounter, n_threads: usize, increments_each: u64) { + todo!("Spawn threads with counter.clone(), join them all") +} + +// ============================================================ +// Exercise 3: Channels — Workers Report Partial Results +// ============================================================ +// +// Python version: +// ```python +// import threading +// from queue import Queue +// +// def sum_of_squares_via_queue(values: list[int], n_workers: int) -> int: +// q = Queue() +// chunk_size = max(1, -(-len(values) // n_workers)) +// chunks = [values[i:i + chunk_size] for i in range(0, len(values), chunk_size)] +// +// def worker(chunk): +// q.put(sum(v * v for v in chunk)) +// +// threads = [threading.Thread(target=worker, args=(c,)) for c in chunks] +// for t in threads: t.start() +// for t in threads: t.join() +// return sum(q.get() for _ in range(len(chunks))) # must count gets! +// ``` +// +// The Rust version uses mpsc::channel. Bonus elegance: when every +// Sender is dropped, the Receiver's iterator ends on its own — you +// don't need to know how many messages to expect. +// +// Hints: +// - `let (tx, rx) = std::sync::mpsc::channel();` +// - spawned threads need owned data: `chunk.to_vec()` each chunk +// - clone `tx` into each thread, send the partial sum +// - `drop(tx)` after spawning, then `rx.iter().sum()` + +pub fn sum_of_squares_via_channel(values: &[i64], n_workers: usize) -> i64 { + todo!("Chunk the values, send each chunk's sum of squares through a channel, sum the receiver") +} + +// ============================================================ +// Exercise 4: rayon — The One-Word Upgrade +// ============================================================ +// +// Python version: +// ```python +// def count_vowels(words: list[str]) -> int: +// return sum(sum(1 for c in w if c in "aeiou") for w in words) +// +// # The "parallel" version is a whole architectural decision: +// from multiprocessing import Pool +// def count_vowels_parallel(words): +// with Pool() as pool: # fork interpreters +// counts = pool.map(count_word, words) # pickle every string +// return sum(counts) +// ``` +// +// In Rust, parallelizing is a one-word diff: `iter()` -> `par_iter()`. +// The sequential version is given; write the parallel one. +// +// Hints: +// - `use rayon::prelude::*;` (inside the function is fine) +// - same body as the sequential version, with `par_iter()` + +pub fn count_vowels_sequential(words: &[String]) -> usize { + words + .iter() + .map(|w| w.chars().filter(|c| "aeiou".contains(*c)).count()) + .sum() +} + +pub fn count_vowels_parallel(words: &[String]) -> usize { + todo!("Same as count_vowels_sequential, but with par_iter") +} + +// ============================================================ +// Exercise 5: rayon Map-Reduce — The Busiest Collatz Number +// ============================================================ +// +// The Collatz sequence: repeatedly apply n -> n/2 (even) or n -> 3n+1 +// (odd) until you reach 1. `collatz_steps` (provided) counts the steps. +// +// Python version: +// ```python +// def busiest_collatz(limit: int) -> tuple[int, int]: +// """Find (n, steps) where n in 1..limit takes the MOST steps. +// Ties go to the smaller n.""" +// best_n, best_steps = 1, 0 +// for n in range(1, limit): +// steps = collatz_steps(n) +// if steps > best_steps: +// best_n, best_steps = n, steps +// return best_n, best_steps +// ``` +// +// Write the rayon version. The interesting part is making ties +// DETERMINISTIC: a parallel max may inspect candidates in any order, so +// "first one wins" is not reproducible. Encode the tie-break in the key +// instead: maximize `(steps, Reverse(n))` so equal step counts prefer +// the smaller n — then every run, sequential or parallel, agrees. +// +// Hints: +// - `(1..limit).into_par_iter()` +// - `.map(|n| (n, collatz_steps(n)))` +// - `.max_by_key(|&(n, steps)| (steps, std::cmp::Reverse(n)))` +// - the range is non-empty for limit >= 2; `.unwrap()` is fine here + +/// Number of Collatz steps to reach 1. Provided — the exercise is the +/// parallel reduction, not the arithmetic. +pub fn collatz_steps(mut n: u64) -> u64 { + let mut steps = 0; + while n > 1 { + n = if n.is_multiple_of(2) { + n / 2 + } else { + 3 * n + 1 + }; + steps += 1; + } + steps +} + +/// Return `(n, steps)` for the n in `1..limit` with the most Collatz +/// steps. Ties prefer the smaller n. +pub fn busiest_collatz(limit: u64) -> (u64, u64) { + todo!("Parallel map n -> (n, steps), then max_by_key with a deterministic tie-break") +} + +// ============================================================ +// Tests — do not modify below this line +// ============================================================ + +#[cfg(test)] +mod tests { + use super::*; + + // Exercise 1 + #[test] + fn ex1_sum_matches_sequential() { + let numbers: Vec = (1..=1000).collect(); + let expected: i64 = numbers.iter().sum(); + assert_eq!(sum_threaded(&numbers, 4), expected); + } + + #[test] + fn ex1_more_chunks_than_numbers() { + let numbers = vec![1, 2, 3]; + assert_eq!(sum_threaded(&numbers, 16), 6); + } + + #[test] + fn ex1_empty_slice() { + assert_eq!(sum_threaded(&[], 4), 0); + } + + // Exercise 2 + #[test] + fn ex2_increment_and_read() { + let counter = SharedCounter::new(); + counter.increment(); + counter.increment(); + counter.increment(); + assert_eq!(counter.value(), 3); + } + + #[test] + fn ex2_clones_share_state() { + let counter = SharedCounter::new(); + let alias = counter.clone(); + counter.increment(); + alias.increment(); + assert_eq!(counter.value(), 2); // both clones hit the same Mutex + } + + #[test] + fn ex2_concurrent_increments_are_not_lost() { + let counter = SharedCounter::new(); + run_incrementers(&counter, 8, 1000); + // Without the Mutex, increments would be lost to races. + // With it: exactly 8 * 1000, every time. + assert_eq!(counter.value(), 8000); + } + + // Exercise 3 + #[test] + fn ex3_matches_sequential() { + let values: Vec = (1..=100).collect(); + let expected: i64 = values.iter().map(|v| v * v).sum(); + assert_eq!(sum_of_squares_via_channel(&values, 4), expected); + } + + #[test] + fn ex3_single_worker() { + let values = vec![3, 4]; + assert_eq!(sum_of_squares_via_channel(&values, 1), 25); + } + + #[test] + fn ex3_empty_input() { + assert_eq!(sum_of_squares_via_channel(&[], 4), 0); + } + + // Exercise 4 + #[test] + fn ex4_parallel_matches_sequential() { + let words: Vec = (0..300) + .map(|i| format!("parallelism-example-{i}")) + .collect(); + assert_eq!( + count_vowels_parallel(&words), + count_vowels_sequential(&words) + ); + } + + #[test] + fn ex4_known_value() { + let words = vec!["aeiou".to_string(), "xyz".to_string(), "rust".to_string()]; + assert_eq!(count_vowels_parallel(&words), 6); // 5 + 0 + 1 + } + + #[test] + fn ex4_empty_input() { + assert_eq!(count_vowels_parallel(&[]), 0); + } + + // Exercise 5 + #[test] + fn ex5_collatz_steps_provided() { + assert_eq!(collatz_steps(1), 0); + assert_eq!(collatz_steps(2), 1); + assert_eq!(collatz_steps(6), 8); // 6→3→10→5→16→8→4→2→1 + assert_eq!(collatz_steps(27), 111); // the famous slow starter + } + + #[test] + fn ex5_busiest_below_30() { + // 27 takes 111 steps — the unique maximum below 30. + assert_eq!(busiest_collatz(30), (27, 111)); + } + + #[test] + fn ex5_trivial_range() { + assert_eq!(busiest_collatz(2), (1, 0)); + } + + #[test] + fn ex5_parallel_matches_sequential_reference() { + use std::cmp::Reverse; + // Sequential reference with the same deterministic tie-break. + let expected = (1..5000u64) + .map(|n| (n, collatz_steps(n))) + .max_by_key(|&(n, steps)| (steps, Reverse(n))) + .unwrap(); + assert_eq!(busiest_collatz(5000), expected); + } +} diff --git a/ch08-escaping-the-gil/src/lib.rs b/ch08-escaping-the-gil/src/lib.rs new file mode 100644 index 0000000..33efc74 --- /dev/null +++ b/ch08-escaping-the-gil/src/lib.rs @@ -0,0 +1,580 @@ +//! # Chapter 8: Escaping the GIL +//! +//! Python threads never run Python bytecode in parallel — the Global +//! Interpreter Lock (GIL) serializes them. Rust threads are real OS +//! threads with no interpreter lock, and the `Send`/`Sync` traits make +//! data races a *compile error* instead of a runtime hazard. +//! +//! Run the tests: `cargo test -p ch08-escaping-the-gil` + +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +use std::thread; + +use rayon::prelude::*; + +// --------------------------------------------------------------------------- +// 1. A CPU-bound workload — the kind Python threads can't speed up +// --------------------------------------------------------------------------- + +/// Trial-division primality test. Deliberately CPU-bound: pure +/// arithmetic, no I/O, nothing for a thread to "wait" on. +/// +/// Python equivalent: +/// ```python +/// def is_prime(n: int) -> bool: +/// if n < 2: +/// return False +/// if n < 4: +/// return True +/// if n % 2 == 0: +/// return False +/// d = 3 +/// while d * d <= n: +/// if n % d == 0: +/// return False +/// d += 2 +/// return True +/// ``` +/// +/// In Python, running this on threads buys you NOTHING: the GIL lets +/// only one thread execute bytecode at a time, so 4 threads of +/// `is_prime` take as long as 1 thread (often longer, due to lock +/// contention). The standard escape hatch is `multiprocessing`, which +/// gets real parallelism by paying a heavy toll: spawning whole +/// interpreter processes and pickling every argument and result across +/// process boundaries. +pub fn is_prime(n: u64) -> bool { + if n < 2 { + return false; + } + if n < 4 { + return true; + } + if n.is_multiple_of(2) { + return false; + } + let mut d = 3; + while d * d <= n { + if n.is_multiple_of(d) { + return false; + } + d += 2; + } + true +} + +/// Count primes in `lo..hi`, sequentially. This is our baseline — every +/// parallel version below must produce exactly this answer. +/// +/// Python equivalent: +/// ```python +/// def count_primes(lo: int, hi: int) -> int: +/// return sum(1 for n in range(lo, hi) if is_prime(n)) +/// ``` +pub fn count_primes(lo: u64, hi: u64) -> usize { + (lo..hi).filter(|&n| is_prime(n)).count() +} + +// --------------------------------------------------------------------------- +// 2. Real threads — std::thread::scope and borrowing across threads +// --------------------------------------------------------------------------- + +/// Count primes by splitting the range across real OS threads. +/// +/// Python equivalent (which does NOT get a speedup, thanks to the GIL): +/// ```python +/// import threading +/// +/// def count_primes_threaded(lo, hi, n_threads): +/// results = [0] * n_threads +/// chunk = -(-(hi - lo) // n_threads) # ceiling division +/// +/// def worker(i): +/// start = min(lo + i * chunk, hi) +/// end = min(start + chunk, hi) +/// results[i] = count_primes(start, end) # all serialized by the GIL +/// +/// threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_threads)] +/// for t in threads: t.start() +/// for t in threads: t.join() +/// return sum(results) +/// ``` +/// +/// The Rust version is structurally identical — but the threads +/// genuinely run in parallel on separate cores. `thread::scope` +/// guarantees every spawned thread finishes before the scope returns, +/// which is why the threads are allowed to *borrow* from the enclosing +/// function (here they only capture small `u64` copies, but they could +/// borrow a `&[u64]` slice too — see the exercises). +pub fn count_primes_threaded(lo: u64, hi: u64, n_threads: usize) -> usize { + let n_threads = n_threads.max(1) as u64; + let chunk = (hi.saturating_sub(lo)).div_ceil(n_threads); + + thread::scope(|s| { + let mut handles = Vec::new(); + for i in 0..n_threads { + let start = (lo + i * chunk).min(hi); + let end = (start + chunk).min(hi); + // Each thread owns its own start/end and returns its own + // count. No shared mutable state — nothing to race on. + handles.push(s.spawn(move || count_primes(start, end))); + } + handles + .into_iter() + .map(|h| h.join().expect("worker thread panicked")) + .sum() + }) +} + +// --------------------------------------------------------------------------- +// 3. Send and Sync — the type system does the GIL's job, without the lock +// --------------------------------------------------------------------------- +// +// Why is sharing data between Python threads "safe"? Because the GIL +// serializes every bytecode instruction — safety by *never actually +// running in parallel*. +// +// Rust takes the opposite deal. Threads really run in parallel, and two +// marker traits decide, at compile time, what may cross a thread +// boundary: +// +// - `Send`: this type can be MOVED to another thread. +// - `Sync`: this type can be SHARED (`&T`) between threads. +// +// You almost never implement these yourself — the compiler derives them +// structurally. `Rc` (non-atomic refcount, like CPython's refcounting +// without the GIL guarding it) is NOT Send. `RefCell` (runtime borrow +// checking, single-threaded) is NOT Sync. Their thread-safe siblings +// are `Arc` (atomic refcount) and `Mutex` (a real lock). + +/// Sum the lengths of all words using threads that SHARE the data. +/// +/// `thread::spawn` (unlike `thread::scope`) may outlive the caller, so +/// it cannot borrow — everything it captures must be `'static` and +/// `Send`. To share one `Vec` among such threads we use `Arc`, the +/// atomically reference-counted pointer. +/// +/// Here is the version that does NOT compile. `Rc`'s reference count is +/// not atomic, so two threads cloning it at once would corrupt the +/// count — the compiler rejects it because `Rc>` is not +/// `Send`: +/// +/// ```compile_fail +/// use std::rc::Rc; +/// use std::thread; +/// +/// let words = Rc::new(vec!["hello".to_string(), "world".to_string()]); +/// let shared = Rc::clone(&words); +/// let handle = thread::spawn(move || shared.len()); // ERROR: `Rc<...>` cannot +/// // be sent between threads +/// handle.join().unwrap(); +/// ``` +/// +/// In Python the equivalent bug is *silent*: every object is happily +/// shared between threads, and the GIL papers over the danger by never +/// letting two threads touch it simultaneously. Rust gives you the +/// parallelism and moves the safety check to the compiler. +/// +/// Python equivalent (shared read-only list — safe only because of the GIL): +/// ```python +/// def total_chars(words: list[str], n_threads: int) -> int: +/// results = [0] * n_threads +/// +/// def worker(i): +/// results[i] = sum(len(w) for w in words[i::n_threads]) +/// +/// threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_threads)] +/// for t in threads: t.start() +/// for t in threads: t.join() +/// return sum(results) +/// ``` +pub fn total_chars_across_threads(words: Vec, n_threads: usize) -> usize { + let n_threads = n_threads.max(1); + let words = Arc::new(words); // atomic refcount: Arc> IS Send + Sync + + let mut handles = Vec::new(); + for i in 0..n_threads { + let words = Arc::clone(&words); // bump the refcount, move the clone in + handles.push(thread::spawn(move || { + // Stride partitioning: thread i takes words[i], words[i+n], ... + words + .iter() + .skip(i) + .step_by(n_threads) + .map(|w| w.len()) + .sum::() + })); + } + handles + .into_iter() + .map(|h| h.join().expect("worker thread panicked")) + .sum() +} + +// --------------------------------------------------------------------------- +// 4. Shared MUTABLE state — Arc> and channels +// --------------------------------------------------------------------------- + +/// Count primes with worker threads adding into one shared counter. +/// +/// Python equivalent: +/// ```python +/// import threading +/// +/// counter = 0 +/// lock = threading.Lock() +/// +/// def worker(start, end): +/// global counter +/// local = count_primes(start, end) # compute OUTSIDE the lock +/// with lock: +/// counter += local # mutate INSIDE the lock +/// ``` +/// +/// The crucial difference: Python's `threading.Lock` and the data it +/// protects are *unrelated objects* — nothing stops a worker from +/// updating `counter` without taking the lock. Rust's `Mutex` OWNS +/// the data. The only way to reach the `usize` inside is through +/// `.lock()`, which returns a guard; the lock releases when the guard +/// goes out of scope (like `with lock:`, but impossible to forget). +/// +/// And here is the race the compiler refuses to compile. `RefCell` is +/// Python-style "checked at runtime, single thread assumed" interior +/// mutability — it is not `Sync`, so you cannot share it across +/// threads even inside an `Arc`: +/// +/// ```compile_fail +/// use std::cell::RefCell; +/// use std::sync::Arc; +/// use std::thread; +/// +/// let counter = Arc::new(RefCell::new(0)); +/// let shared = Arc::clone(&counter); +/// let handle = thread::spawn(move || { +/// *shared.borrow_mut() += 1; // ERROR: `RefCell` cannot be +/// // shared between threads safely +/// }); +/// handle.join().unwrap(); +/// ``` +/// +/// Swap `RefCell` for `Mutex` and it compiles — that one-word change is +/// the whole single-threaded/multi-threaded migration, enforced by types. +pub fn count_primes_shared_counter(lo: u64, hi: u64, n_threads: usize) -> usize { + let n_threads = n_threads.max(1) as u64; + let chunk = (hi.saturating_sub(lo)).div_ceil(n_threads); + let counter = Arc::new(Mutex::new(0usize)); + + let mut handles = Vec::new(); + for i in 0..n_threads { + let counter = Arc::clone(&counter); + let start = (lo + i * chunk).min(hi); + let end = (start + chunk).min(hi); + handles.push(thread::spawn(move || { + let local = count_primes(start, end); // compute outside the lock + *counter.lock().expect("mutex poisoned") += local; // brief critical section + })); + } + for handle in handles { + handle.join().expect("worker thread panicked"); + } + + let total = *counter.lock().expect("mutex poisoned"); + total +} + +/// Count primes using a channel instead of a shared counter. +/// +/// `mpsc::channel` is Rust's `queue.Queue`: multi-producer, +/// single-consumer, and the receiving end doubles as an iterator. +/// +/// Python equivalent: +/// ```python +/// from queue import Queue +/// +/// def count_primes_with_queue(lo, hi, n_threads): +/// q = Queue() +/// chunk = -(-(hi - lo) // n_threads) +/// +/// def worker(start, end): +/// q.put(count_primes(start, end)) +/// +/// threads = [] +/// for i in range(n_threads): +/// start = min(lo + i * chunk, hi) +/// end = min(start + chunk, hi) +/// t = threading.Thread(target=worker, args=(start, end)) +/// t.start() +/// threads.append(t) +/// for t in threads: t.join() +/// return sum(q.get() for _ in range(n_threads)) +/// ``` +/// +/// Channels often beat shared state: each value has exactly ONE owner +/// at a time, ownership transfers through the channel, and there is no +/// lock to hold wrong. "Do not communicate by sharing memory; share +/// memory by communicating." +pub fn count_primes_channel(lo: u64, hi: u64, n_threads: usize) -> usize { + let n_threads = n_threads.max(1) as u64; + let chunk = (hi.saturating_sub(lo)).div_ceil(n_threads); + let (tx, rx) = mpsc::channel(); + + for i in 0..n_threads { + let tx = tx.clone(); // multi-producer: every worker gets a sender + let start = (lo + i * chunk).min(hi); + let end = (start + chunk).min(hi); + thread::spawn(move || { + tx.send(count_primes(start, end)).expect("receiver dropped"); + }); + } + drop(tx); // drop the original sender so the iterator below terminates + + // rx.iter() yields until every sender is dropped — no sentinel + // values, no "poison pill" pattern, no counting how many to expect. + rx.iter().sum() +} + +// --------------------------------------------------------------------------- +// 5. rayon — the "free" upgrade from sequential to parallel +// --------------------------------------------------------------------------- + +/// Count primes in parallel with rayon. Compare to `count_primes`: +/// the ONLY change is `iter` → `par_iter` (`into_par_iter` for ranges). +/// +/// Python equivalent: +/// ```python +/// from multiprocessing import Pool +/// +/// def count_primes_parallel(lo, hi): +/// with Pool() as pool: +/// flags = pool.map(is_prime, range(lo, hi)) # pickles every int +/// return sum(flags) # across process pipes +/// ``` +/// +/// `multiprocessing.Pool` is Python's honest workaround for the GIL — +/// but it forks whole interpreter processes, pickles every argument, +/// and pickles every result back. rayon's work-stealing thread pool +/// shares the address space: no copies, no serialization, and chunking +/// is automatic. The sequential and parallel versions are guaranteed to +/// produce identical results, so the tests assert exactly that. +pub fn count_primes_rayon(lo: u64, hi: u64) -> usize { + (lo..hi).into_par_iter().filter(|&n| is_prime(n)).count() +} + +/// A map/reduce over a slice: total characters across all words. +/// +/// Python equivalent: +/// ```python +/// def total_chars(words: list[str]) -> int: +/// return sum(len(w) for w in words) +/// +/// # The multiprocessing version must pickle every string to the workers: +/// with Pool() as pool: +/// total = sum(pool.map(len, words)) +/// ``` +/// +/// With rayon, `.par_iter()` borrows the slice in place — zero copies. +/// `Send`/`Sync` still guard the door: if the item type were not safe +/// to share across threads, `.par_iter()` would not compile. +pub fn total_chars_rayon(words: &[String]) -> usize { + words.par_iter().map(|w| w.len()).sum() +} + +/// The same computation, sequentially — kept for the tests to compare +/// against, and to show how little changes: `iter` vs `par_iter`. +pub fn total_chars_sequential(words: &[String]) -> usize { + words.iter().map(|w| w.len()).sum() +} + +// --------------------------------------------------------------------------- +// 6. The FFI payoff — releasing the GIL from a PyO3 extension +// --------------------------------------------------------------------------- +// +// Here is where the whole course arc pays off. A PyO3 extension module +// holds the GIL while it talks to Python objects — but pure-Rust work +// doesn't need Python at all. `py.allow_threads(...)` RELEASES the GIL +// for the duration of a closure, letting: +// +// 1. other Python threads keep running while Rust crunches numbers, and +// 2. the Rust code inside fan out across every core with rayon. +// +// The closure cannot touch any Python object (PyO3 enforces this with — +// you guessed it — `Send` bounds), so it is provably safe to let the +// interpreter carry on without us. +// +// From Python, the result looks like magic: +// +// ```python +// import threading +// from my_extension import count_primes_releasing_gil +// +// # This call saturates every core via rayon, and while it runs, OTHER +// # Python threads keep executing — the GIL is released for the duration. +// t = threading.Thread(target=count_primes_releasing_gil, args=(2, 10_000_000)) +// t.start() +// do_other_python_work() # not blocked! +// t.join() +// ``` +// +// Python's weakness (threads can't compute in parallel) becomes a +// non-issue: you keep writing Python, and the hot loop escapes to Rust +// threads that the GIL cannot see. +// +// This module only compiles with `--features python` so that the +// default build and CI need no Python installation. + +#[cfg(feature = "python")] +pub mod python { + use pyo3::prelude::*; + + /// Count primes in `lo..hi` on all cores, with the GIL released. + /// + /// The signature takes `py: Python<'_>` — the token that PROVES we + /// hold the GIL. `allow_threads` consumes that proof, releases the + /// lock, runs the closure on plain Rust data, and re-acquires the + /// GIL before returning to Python. + #[pyfunction] + pub fn count_primes_releasing_gil(py: Python<'_>, lo: u64, hi: u64) -> usize { + py.allow_threads(|| super::count_primes_rayon(lo, hi)) + } + + /// The module definition: `import ch08_escaping_the_gil` from Python + /// (build the wheel with `maturin`, as covered in Chapter 6). + #[pymodule] + fn ch08_escaping_the_gil(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(count_primes_releasing_gil, m)?)?; + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- +// +// Note what these tests do NOT assert: timing. Parallel code is tested +// for CORRECTNESS (parallel result == sequential result) because +// wall-clock speedups depend on core counts and noisy schedulers — +// especially on shared CI runners. Discuss performance in prose; +// assert determinism in tests. + +#[cfg(test)] +mod tests { + use super::*; + + // There are 168 primes below 1000 — a classic checkable constant. + const PRIMES_BELOW_1000: usize = 168; + + // The CPU-bound baseline + + #[test] + fn is_prime_basics() { + assert!(!is_prime(0)); + assert!(!is_prime(1)); + assert!(is_prime(2)); + assert!(is_prime(3)); + assert!(!is_prime(4)); + assert!(is_prime(97)); + assert!(!is_prime(1_000_000)); + assert!(is_prime(1_000_003)); + } + + #[test] + fn sequential_baseline() { + assert_eq!(count_primes(0, 1000), PRIMES_BELOW_1000); + assert_eq!(count_primes(0, 10), 4); // 2, 3, 5, 7 + assert_eq!(count_primes(10, 10), 0); // empty range + assert_eq!(count_primes(20, 10), 0); // inverted range + } + + // Scoped threads + + #[test] + fn threaded_matches_sequential() { + assert_eq!(count_primes_threaded(0, 1000, 4), PRIMES_BELOW_1000); + } + + #[test] + fn threaded_with_more_threads_than_work() { + // 16 threads over 10 numbers: most chunks are empty, answer unchanged. + assert_eq!(count_primes_threaded(0, 10, 16), 4); + } + + #[test] + fn threaded_handles_zero_threads() { + // Degenerate input is clamped to one thread, not a panic. + assert_eq!(count_primes_threaded(0, 100, 0), count_primes(0, 100)); + } + + // Arc — shared read-only data across spawned threads + + #[test] + fn arc_shared_data_matches_sequential() { + let words: Vec = ["alpha", "beta", "gamma", "delta", "epsilon"] + .iter() + .map(|s| s.to_string()) + .collect(); + let expected: usize = words.iter().map(|w| w.len()).sum(); + assert_eq!(total_chars_across_threads(words, 3), expected); + } + + #[test] + fn arc_shared_data_empty_input() { + assert_eq!(total_chars_across_threads(Vec::new(), 4), 0); + } + + // Arc> — shared mutable state + + #[test] + fn shared_counter_matches_sequential() { + assert_eq!(count_primes_shared_counter(0, 1000, 4), PRIMES_BELOW_1000); + } + + #[test] + fn shared_counter_single_thread() { + assert_eq!(count_primes_shared_counter(0, 1000, 1), PRIMES_BELOW_1000); + } + + // Channels + + #[test] + fn channel_matches_sequential() { + assert_eq!(count_primes_channel(0, 1000, 4), PRIMES_BELOW_1000); + } + + #[test] + fn channel_empty_range() { + assert_eq!(count_primes_channel(100, 100, 4), 0); + } + + // rayon + + #[test] + fn rayon_matches_sequential() { + // The load-bearing assertion of the chapter: parallel and + // sequential produce IDENTICAL results. + assert_eq!(count_primes_rayon(0, 10_000), count_primes(0, 10_000)); + } + + #[test] + fn rayon_known_value() { + assert_eq!(count_primes_rayon(0, 1000), PRIMES_BELOW_1000); + } + + #[test] + fn rayon_map_reduce_matches_sequential() { + let words: Vec = (0..500).map(|i| format!("word-{i}")).collect(); + assert_eq!(total_chars_rayon(&words), total_chars_sequential(&words)); + } + + #[test] + fn all_strategies_agree() { + // Five implementations, one answer. This is the property that + // makes parallelism safe to adopt: same inputs, same outputs, + // regardless of scheduling. + let expected = count_primes(0, 2000); + assert_eq!(count_primes_threaded(0, 2000, 4), expected); + assert_eq!(count_primes_shared_counter(0, 2000, 4), expected); + assert_eq!(count_primes_channel(0, 2000, 4), expected); + assert_eq!(count_primes_rayon(0, 2000), expected); + } +} From 05f7adc0dc1b5ca8f4519740c3bd2b866027611f Mon Sep 17 00:00:00 2001 From: hartsock Date: Fri, 5 Jun 2026 21:49:10 -0400 Subject: [PATCH 5/5] docs: move chapters 5-8 from roadmap to shipped chapters table Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1d982ac..200c3b5 100644 --- a/README.md +++ b/README.md @@ -34,19 +34,19 @@ giving up everything that makes Python productive. | 2 | **Error Handling** | `try/except`, `None`, `LBYL vs EAFP` | `Result`, `Option`, `?` operator | | 3 | **Traits & Generics** | ABCs, Protocols, duck typing | Traits, generics, trait bounds | | 4 | **Content-Addressable Data** | `hashlib`, `json.dumps(sort_keys=True)` | BLAKE3, serde, CIDs, deterministic serialization | +| 5 | **CLI Tools** | `argparse`, `click` | `clap`, structured output, self-documenting tools | +| 6 | **FFI & PyO3** | `ctypes`, C extensions, wheels | PyO3 bindings, `maturin`, packaging Rust as a Python module | +| 7 | **PyO3 House Style** | "how should this feel from Python?" | feature-gated bindings, error mapping to exceptions, zero-copy data exchange | +| 8 | **Escaping the GIL** | `threading` vs `multiprocessing`, the GIL | `Send`/`Sync`, `rayon`, releasing the GIL across FFI calls | ## Course Roadmap -The arc from here bends toward the interop boundary — each planned chapter -takes one thing Python developers wish they had and shows how Rust provides -it *to* Python, not instead of it: +The arc continues toward the interop boundary — each planned chapter takes +one thing Python developers wish they had and shows how Rust provides it +*to* Python, not instead of it: | # | Title | Python Concept | Rust Concept | |---|-------|---------------|--------------| -| 5 | **CLI Tools** | `argparse`, `click` | `clap`, structured output, self-documenting tools | -| 6 | **FFI & PyO3** | `ctypes`, C extensions, wheels | PyO3 bindings, `maturin`, packaging Rust as a Python module | -| 7 | **PyO3 House Style** | "how should this feel from Python?" | feature-gated bindings, error mapping to exceptions, zero-copy data exchange | -| 8 | **Escaping the GIL** | `threading` vs `multiprocessing`, the GIL | `Send`/`Sync`, `rayon`, releasing the GIL across FFI calls | | 9 | **Types That Travel** | type hints, `mypy`, runtime validation | newtypes, exhaustive enums, type-safe IDs surfacing as Python types | | 10 | **Case Study: Speed Where It Counts** | profiling a real hot path | swapping it for a published Rust crate and measuring the win |