From 76953b7465de5183609fbc8a0abeeacb4a84009b Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 22:07:55 +0300 Subject: [PATCH 01/14] feat(codegen): verify generated output drift --- .github/workflows/pr.yaml | 5 +- Justfile | 5 + codegen/src/main.rs | 5 + codegen/src/targets/mod.rs | 7 ++ codegen/src/targets/write.rs | 135 +++++++++++++++++++++++++-- docs/src/contributing/development.md | 9 ++ 6 files changed, 156 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9c8828d..aa9a8b8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -71,10 +71,7 @@ jobs: run: uv sync --project targets/ptfkit-py --frozen --all-groups - name: Check generated files - run: | - cargo run --manifest-path codegen/Cargo.toml generate - git diff --exit-code - test -z "$(git status --porcelain)" + run: cargo run --manifest-path codegen/Cargo.toml check-generated - name: Check codegen formatting id: codegen-format diff --git a/Justfile b/Justfile index bfc7130..ef23761 100644 --- a/Justfile +++ b/Justfile @@ -18,6 +18,11 @@ mod rust 'targets/ptfkit-rs' @generate: cargo run generate +# Regenerate all codegen-owned files and fail if any output drifts. +[working-directory: 'codegen'] +@check-generated: + cargo run check-generated + # Set the package version and refresh dependent lockfiles. @version value: cargo run --manifest-path codegen/Cargo.toml -- version {{quote(value)}} diff --git a/codegen/src/main.rs b/codegen/src/main.rs index e4583a3..50b0cf8 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -24,6 +24,7 @@ pub(crate) struct Cli { enum Command { Validate, Generate, + CheckGenerated, Version { version: String }, } @@ -50,6 +51,10 @@ impl Cli { let entries = load_validated_specifications(root)?; targets::run(root, entries) } + Command::CheckGenerated => { + let entries = load_validated_specifications(root)?; + targets::check_generated(root, entries) + } Command::Version { version } => version::run(root, &version), } } diff --git a/codegen/src/targets/mod.rs b/codegen/src/targets/mod.rs index 37d6a5f..6a4a57d 100644 --- a/codegen/src/targets/mod.rs +++ b/codegen/src/targets/mod.rs @@ -226,6 +226,13 @@ pub(crate) fn run(root: &Path, entries: Vec) -> Result<()> { ) } +/// Regenerate every target and fail when that changes a codegen-owned file. +pub(crate) fn check_generated(root: &Path, entries: Vec) -> Result<()> { + let before = write::snapshot_generated(root)?; + run(root, entries)?; + write::assert_unchanged(root, before) +} + #[cfg(test)] mod tests { use std::path::Path; diff --git a/codegen/src/targets/write.rs b/codegen/src/targets/write.rs index a3726e1..1635e44 100644 --- a/codegen/src/targets/write.rs +++ b/codegen/src/targets/write.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeSet, + collections::{BTreeMap, BTreeSet}, fs::{self, OpenOptions}, io::Write, path::{Path, PathBuf}, @@ -19,6 +19,99 @@ struct StagedWrite { static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); +pub(super) struct GeneratedTree(BTreeMap>); + +pub(super) fn snapshot_generated(root: &Path) -> Result { + let mut files = BTreeMap::new(); + for target in Target::ALL { + collect_generated( + root, + &target.cleanup_directory(root), + target.generated_header(), + &mut files, + )?; + } + Ok(GeneratedTree(files)) +} + +pub(super) fn assert_unchanged(root: &Path, before: GeneratedTree) -> Result<()> { + let after = snapshot_generated(root)?; + if let Some(report) = drift_report(&before.0, &after.0) { + bail!("generated output drift after regeneration:\n{report}") + } + println!("generated output is unchanged"); + Ok(()) +} + +fn collect_generated( + root: &Path, + directory: &Path, + header: &str, + files: &mut BTreeMap>, +) -> Result<()> { + if !directory.is_dir() { + return Ok(()); + } + for entry in fs::read_dir(directory)? { + let path = entry?.path(); + if path.is_dir() { + collect_generated(root, &path, header, files)?; + } else { + let contents = fs::read(&path)?; + if is_generated(&contents, header) { + let relative = path + .strip_prefix(root) + .context("finding generated-file path relative to the workspace root")? + .to_owned(); + files.insert(relative, contents); + } + } + } + Ok(()) +} + +fn drift_report( + before: &BTreeMap>, + after: &BTreeMap>, +) -> Option { + let added = after + .keys() + .filter(|path| !before.contains_key(*path)) + .collect::>(); + let removed = before + .keys() + .filter(|path| !after.contains_key(*path)) + .collect::>(); + let modified = before + .iter() + .filter(|(path, contents)| after.get(*path).is_some_and(|after| after != *contents)) + .map(|(path, _)| path) + .collect::>(); + + if added.is_empty() && removed.is_empty() && modified.is_empty() { + return None; + } + + let mut report = String::new(); + append_paths(&mut report, "added", &added); + append_paths(&mut report, "removed", &removed); + append_paths(&mut report, "modified", &modified); + Some(report.trim_end().to_owned()) +} + +fn append_paths(report: &mut String, label: &str, paths: &[&PathBuf]) { + if paths.is_empty() { + return; + } + report.push_str(label); + report.push_str(":\n"); + for path in paths { + report.push_str(" "); + report.push_str(&path.display().to_string()); + report.push('\n'); + } +} + pub(super) fn commit(root: &Path, outputs: &[TargetOutput]) -> Result<()> { let staged = stage(root, outputs)?; if let Err(error) = format(root, &staged) { @@ -120,10 +213,12 @@ fn remove_obsolete(directory: &Path, expected: &BTreeSet, header: &str) fn is_generated(contents: &[u8], header: &str) -> bool { contents.starts_with(header.as_bytes()) || (header == super::documentation::HEADER - && contents.starts_with(b"---\n") - && contents - .windows(header.len()) - .any(|window| window == header.as_bytes())) + && (contents.strip_prefix(b"---\n").is_some_and(|contents| { + contents.starts_with(super::documentation::FRONTMATTER_HEADER.as_bytes()) + }) || (contents.starts_with(b"---\n") + && contents + .windows(header.len()) + .any(|window| window == header.as_bytes())))) } fn format(root: &Path, staged: &[StagedWrite]) -> Result<()> { @@ -203,7 +298,7 @@ fn remove_temporary(staged: &[StagedWrite]) { #[cfg(test)] mod tests { - use std::{fs, path::Path}; + use std::{collections::BTreeMap, fs, path::Path}; use super::*; use crate::targets::{GeneratedFile, documentation::HEADER}; @@ -255,7 +350,35 @@ mod tests { format!("---\ntitle: \"ptfkit.test\"\n---\n\n{HEADER}::: ptfkit.test\n").as_bytes(), HEADER, )); + assert!(is_generated( + format!( + "---\n{}title: ptfkit.test\n---\n", + super::super::documentation::FRONTMATTER_HEADER + ) + .as_bytes(), + HEADER, + )); assert!(!is_generated(b"# Handwritten page\n", HEADER)); assert!(Path::new("index.md").is_relative()); } + + #[test] + fn generated_output_drift_reports_added_removed_and_modified_files() { + let before = BTreeMap::from([ + (PathBuf::from("generated/modified"), b"before".to_vec()), + (PathBuf::from("generated/removed"), b"removed".to_vec()), + ]); + let after = BTreeMap::from([ + (PathBuf::from("generated/added"), b"added".to_vec()), + (PathBuf::from("generated/modified"), b"after".to_vec()), + ]); + + assert_eq!( + drift_report(&before, &after), + Some( + "added:\n generated/added\nremoved:\n generated/removed\nmodified:\n generated/modified" + .to_owned() + ) + ); + } } diff --git a/docs/src/contributing/development.md b/docs/src/contributing/development.md index 6f6df4e..dc04efa 100644 --- a/docs/src/contributing/development.md +++ b/docs/src/contributing/development.md @@ -56,6 +56,15 @@ just generate A second generation run must leave the working tree unchanged. +To check this across every codegen-owned target family, run: + +```sh +just check-generated +``` + +The command regenerates the targets through the normal pipeline and reports +added, removed, or modified generated files. + ### Generation conventions The specification filename stem is the APA-style source slug. Codegen uses it From ba71b14d8592df927f36b6410632dfecf3854eb9 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 22:19:59 +0300 Subject: [PATCH 02/14] refactor(codegen): add rendering foundation --- codegen/src/main.rs | 1 + codegen/src/render.rs | 133 ++++++++++++++++++++ codegen/src/targets/python_documentation.rs | 66 +++++++--- 3 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 codegen/src/render.rs diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 50b0cf8..6febf84 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -7,6 +7,7 @@ use clap::{Parser, Subcommand}; mod formula; mod model; +mod render; mod semantic; mod specs; mod targets; diff --git a/codegen/src/render.rs b/codegen/src/render.rs new file mode 100644 index 0000000..d872eb4 --- /dev/null +++ b/codegen/src/render.rs @@ -0,0 +1,133 @@ +use std::fmt::{self, Display, Write as _}; + +/// Renders a structured value into a [`Writer`]. +pub(crate) trait Render { + fn render(&self, writer: &mut Writer); +} + +/// A small, indentation-aware buffer for textual generators. +pub(crate) struct Writer { + contents: String, + indentation: usize, + at_line_start: bool, +} + +impl Writer { + pub(crate) fn new() -> Self { + Self { + contents: String::new(), + indentation: 0, + at_line_start: true, + } + } + + pub(crate) fn write(&mut self, value: impl Display) { + self.write_fmt(format_args!("{value}")) + .expect("writing to a String cannot fail"); + } + + pub(crate) fn line(&mut self, value: impl Display) { + self.write(value); + self.write("\n"); + } + + pub(crate) fn blank_line(&mut self) { + if !self.at_line_start { + self.write("\n"); + } + self.write("\n"); + } + + pub(crate) fn indented(&mut self, render: impl FnOnce(&mut Self)) { + self.indentation += 1; + render(self); + self.indentation -= 1; + } + + pub(crate) fn block( + &mut self, + opening: impl Display, + indent_contents: bool, + render: impl FnOnce(&mut Self), + closing: impl Display, + ) { + self.line(opening); + if indent_contents { + self.indented(render); + } else { + render(self); + } + self.line(closing); + } + + pub(crate) fn into_string(self) -> String { + self.contents + } +} + +impl fmt::Write for Writer { + fn write_str(&mut self, text: &str) -> fmt::Result { + for part in text.split_inclusive('\n') { + if self.at_line_start && part != "\n" { + for _ in 0..self.indentation { + self.contents.push_str(" "); + } + } + self.contents.push_str(part); + self.at_line_start = part.ends_with('\n'); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{Render, Writer}; + + struct Paragraph(&'static str); + + impl Render for Paragraph { + fn render(&self, writer: &mut Writer) { + writer.line(self.0); + } + } + + #[test] + fn writes_display_values_and_blank_lines() { + let mut writer = Writer::new(); + writer.write("value: "); + writer.line(42); + writer.blank_line(); + writer.line("next"); + + assert_eq!(writer.into_string(), "value: 42\n\nnext\n"); + } + + #[test] + fn renders_nested_blocks_with_indentation() { + let mut writer = Writer::new(); + writer.block( + "outer {", + true, + |writer| { + writer.line("first;"); + writer.block("inner {", true, |writer| writer.line("second;"), "}"); + }, + "}", + ); + + assert_eq!( + writer.into_string(), + "outer {\n first;\n inner {\n second;\n }\n}\n" + ); + } + + #[test] + fn composes_render_values() { + let mut writer = Writer::new(); + Paragraph("first").render(&mut writer); + Paragraph("second").render(&mut writer); + + assert_eq!(writer.into_string(), "first\nsecond\n"); + } +} diff --git a/codegen/src/targets/python_documentation.rs b/codegen/src/targets/python_documentation.rs index 3d2408c..d7557a6 100644 --- a/codegen/src/targets/python_documentation.rs +++ b/codegen/src/targets/python_documentation.rs @@ -1,6 +1,9 @@ use std::path::PathBuf; -use crate::model::Entry; +use crate::{ + model::Entry, + render::{Render, Writer}, +}; use super::{GeneratedFile, documentation::FRONTMATTER_HEADER, py::natural_sort_key}; @@ -20,28 +23,57 @@ fn file(path: impl Into, contents: String) -> GeneratedFile { } fn index(entries: &[&Entry]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: Python API reference\n---\n\n"); - text.push_str("# Python API reference\n\n"); - text.push_str("ptfkit's Python API is organized around public source modules.\n\n"); - text.push_str("## Modules\n\n"); + let mut writer = Writer::new(); + writer.block( + "---", + false, + |writer| { + writer.write(FRONTMATTER_HEADER); + writer.line("title: Python API reference"); + }, + "---", + ); + writer.blank_line(); + writer.line("# Python API reference"); + writer.blank_line(); + writer.line("ptfkit's Python API is organized around public source modules."); + writer.blank_line(); + writer.line("## Modules"); + writer.blank_line(); for entry in entries { - text.push_str(&format!( - "- [`ptfkit.{slug}`]({slug}.md) — {summary}\n", - slug = entry.slug, - summary = entry.spec.source.summary - )); + ModuleReference { entry }.render(&mut writer); } - text + writer.into_string() } fn page(slug: &str) -> GeneratedFile { let module = format!("ptfkit.{slug}"); - file( - format!("{slug}.md"), - format!( - "---\n{FRONTMATTER_HEADER}title: Python module {module}\nnav-title: {module}\n---\n\n::: {module}" - ), - ) + let mut writer = Writer::new(); + writer.block( + "---", + false, + |writer| { + writer.write(FRONTMATTER_HEADER); + writer.line(format_args!("title: Python module {module}")); + writer.line(format_args!("nav-title: {module}")); + }, + "---", + ); + writer.blank_line(); + writer.line(format_args!("::: {module}")); + file(format!("{slug}.md"), writer.into_string()) +} + +struct ModuleReference<'a> { + entry: &'a Entry, +} + +impl Render for ModuleReference<'_> { + fn render(&self, writer: &mut Writer) { + let slug = &self.entry.slug; + let summary = &self.entry.spec.source.summary; + writer.line(format_args!("- [`ptfkit.{slug}`]({slug}.md) — {summary}")); + } } #[cfg(test)] From 384369fbe045d5d1444ca87d00618893f21a0f31 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 22:30:35 +0300 Subject: [PATCH 03/14] refactor(codegen): render C expressions directly --- codegen/src/targets/c_expression.rs | 347 +++++++++++++++++++--------- codegen/src/targets/native.rs | 6 +- codegen/src/targets/py/c.rs | 2 +- 3 files changed, 237 insertions(+), 118 deletions(-) diff --git a/codegen/src/targets/c_expression.rs b/codegen/src/targets/c_expression.rs index cfd4df6..895d96d 100644 --- a/codegen/src/targets/c_expression.rs +++ b/codegen/src/targets/c_expression.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use std::fmt; use crate::semantic::{BinaryOp, Expr, MathFunction, Reference, UnaryOp, Variable}; @@ -13,8 +13,14 @@ pub(super) fn render( inputs: &[String], variables: &[Variable], dialect: Dialect, -) -> Result { - Ok(rendered(expression, inputs, variables, dialect)?.text) +) -> String { + Expression { + expression, + inputs, + variables, + dialect, + } + .to_string() } pub(super) fn float_literal(lexeme: &str) -> String { @@ -48,120 +54,158 @@ enum Precedence { Primary, } -struct RenderedExpression { - text: String, - precedence: Precedence, +struct Expression<'a> { + expression: &'a Expr, + inputs: &'a [String], + variables: &'a [Variable], + dialect: Dialect, } -impl RenderedExpression { - fn binary_operand(&self, parent: Precedence, is_right: bool) -> String { - let needs_parentheses = self.precedence < parent || (is_right && self.precedence == parent); - if needs_parentheses { - format!("({})", self.text) - } else { - self.text.clone() - } +impl fmt::Display for Expression<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.write_expression(formatter, self.expression, None) } +} - fn unary_operand(&self) -> String { - if self.precedence <= Precedence::Unary { - format!("({})", self.text) - } else { - self.text.clone() +impl Expression<'_> { + fn write_expression( + &self, + formatter: &mut fmt::Formatter<'_>, + expression: &Expr, + parent: Option<(Precedence, bool)>, + ) -> fmt::Result { + if let Expr::Unary { + op: UnaryOp::Plus, + operand, + } = expression + { + return self.write_expression(formatter, operand, parent); } - } -} -fn rendered( - expression: &Expr, - inputs: &[String], - variables: &[Variable], - dialect: Dialect, -) -> Result { - Ok(match expression { - Expr::Number(number) => primary(float_literal(&number.lexeme)), - Expr::Reference(Reference::Input(index)) => primary(inputs[*index].clone()), - Expr::Reference(Reference::Variable(index)) => primary(variables[*index].name.clone()), - Expr::Unary { op, operand } => match op { - UnaryOp::Plus => rendered(operand, inputs, variables, dialect)?, - UnaryOp::Minus => { - let operand = rendered(operand, inputs, variables, dialect)?.unary_operand(); - RenderedExpression { - text: format!("-{operand}"), - precedence: Precedence::Unary, - } + let precedence = precedence(expression); + let parenthesize = parent.is_some_and(|(parent, is_right)| { + precedence < parent || (is_right && precedence == parent) + }); + if parenthesize { + write!(formatter, "(")?; + } + + match expression { + Expr::Number(number) => write!(formatter, "{}", float_literal(&number.lexeme))?, + Expr::Reference(Reference::Input(index)) => { + write!(formatter, "{}", self.inputs[*index])? } - }, - Expr::Binary { op, left, right } => { - let left = rendered(left, inputs, variables, dialect)?; - let right = rendered(right, inputs, variables, dialect)?; - match op { - BinaryOp::Add => binary(left, right, Precedence::Sum, "+"), - BinaryOp::Subtract => binary(left, right, Precedence::Sum, "-"), - BinaryOp::Multiply => binary(left, right, Precedence::Product, "*"), - BinaryOp::Divide => binary(left, right, Precedence::Product, "/"), - BinaryOp::Power => primary(format!( - "{}({}, {})", - math_name("pow", dialect), - left.text, - right.text - )), + Expr::Reference(Reference::Variable(index)) => { + write!(formatter, "{}", self.variables[*index].name)?; + } + Expr::Unary { op, operand } => match op { + UnaryOp::Plus => unreachable!("unary plus is handled before parenthesizing"), + UnaryOp::Minus => { + write!(formatter, "-")?; + self.write_expression(formatter, operand, Some((Precedence::Unary, true)))?; + } + }, + Expr::Binary { op, left, right } => match op { + BinaryOp::Add => self.write_binary(formatter, left, right, Precedence::Sum, "+")?, + BinaryOp::Subtract => { + self.write_binary(formatter, left, right, Precedence::Sum, "-")? + } + BinaryOp::Multiply => { + self.write_binary(formatter, left, right, Precedence::Product, "*")? + } + BinaryOp::Divide => { + self.write_binary(formatter, left, right, Precedence::Product, "/")? + } + BinaryOp::Power => { + write!(formatter, "{}(", self.math_name("pow"))?; + self.write_expression(formatter, left, None)?; + write!(formatter, ", ")?; + self.write_expression(formatter, right, None)?; + write!(formatter, ")")?; + } + }, + Expr::Call { function, args } => { + write!(formatter, "{}(", self.function_name(*function))?; + for (index, argument) in args.iter().enumerate() { + if index > 0 { + write!(formatter, ", ")?; + } + self.write_expression(formatter, argument, None)?; + } + write!(formatter, ")")?; } } - Expr::Call { function, args } => { - let args = args - .iter() - .map(|arg| rendered(arg, inputs, variables, dialect)) - .collect::>>()?; - let name = match function { - MathFunction::Sqrt => "sqrt", - MathFunction::Exp => "exp", - MathFunction::Ln => "log", - MathFunction::Log10 => "log10", - MathFunction::Abs => "fabs", - MathFunction::Min => "fmin", - MathFunction::Max => "fmax", - }; - let name = match (dialect, function) { - (Dialect::Cpp, MathFunction::Abs) => "std::abs".to_owned(), - _ => math_name(name, dialect), - }; - primary(format!( - "{name}({})", - args.iter() - .map(|argument| argument.text.as_str()) - .collect::>() - .join(", ") - )) + + if parenthesize { + write!(formatter, ")")?; } - }) -} + Ok(()) + } -fn math_name(name: &str, dialect: Dialect) -> String { - match dialect { - Dialect::C => name.to_owned(), - Dialect::Cpp => format!("std::{name}"), + fn write_binary( + &self, + formatter: &mut fmt::Formatter<'_>, + left: &Expr, + right: &Expr, + precedence: Precedence, + operator: &str, + ) -> fmt::Result { + self.write_expression(formatter, left, Some((precedence, false)))?; + write!(formatter, " {operator} ")?; + self.write_expression(formatter, right, Some((precedence, true))) } -} -fn primary(text: String) -> RenderedExpression { - RenderedExpression { - text, - precedence: Precedence::Primary, + fn function_name(&self, function: MathFunction) -> &'static str { + match function { + MathFunction::Sqrt => self.math_name("sqrt"), + MathFunction::Exp => self.math_name("exp"), + MathFunction::Ln => self.math_name("log"), + MathFunction::Log10 => self.math_name("log10"), + MathFunction::Abs if matches!(self.dialect, Dialect::Cpp) => "std::abs", + MathFunction::Abs => self.math_name("fabs"), + MathFunction::Min => self.math_name("fmin"), + MathFunction::Max => self.math_name("fmax"), + } + } + + fn math_name(&self, name: &'static str) -> &'static str { + match self.dialect { + Dialect::C => name, + Dialect::Cpp => match name { + "sqrt" => "std::sqrt", + "exp" => "std::exp", + "log" => "std::log", + "log10" => "std::log10", + "fabs" => "std::fabs", + "fmin" => "std::fmin", + "fmax" => "std::fmax", + "pow" => "std::pow", + _ => unreachable!("unsupported C/C++ math function"), + }, + } } } -fn binary( - left: RenderedExpression, - right: RenderedExpression, - precedence: Precedence, - operator: &str, -) -> RenderedExpression { - let left = left.binary_operand(precedence, false); - let right = right.binary_operand(precedence, true); - RenderedExpression { - text: format!("{left} {operator} {right}"), - precedence, +fn precedence(expression: &Expr) -> Precedence { + match expression { + Expr::Binary { + op: BinaryOp::Add | BinaryOp::Subtract, + .. + } => Precedence::Sum, + Expr::Binary { + op: BinaryOp::Multiply | BinaryOp::Divide, + .. + } => Precedence::Product, + Expr::Unary { + op: UnaryOp::Minus, .. + } => Precedence::Unary, + Expr::Unary { + op: UnaryOp::Plus, + operand, + } => precedence(operand), + Expr::Number(_) | Expr::Reference(_) | Expr::Binary { .. } | Expr::Call { .. } => { + Precedence::Primary + } } } @@ -170,11 +214,16 @@ mod tests { use super::*; use crate::semantic::{BinaryOp, Reference}; + fn inputs() -> Vec { + vec!["x".into(), "y".into(), "z".into()] + } + + fn input(index: usize) -> Expr { + Expr::Reference(Reference::Input(index)) + } + #[test] fn preserves_precedence_for_c_and_cpp() { - let inputs = vec!["x".into(), "y".into(), "z".into()]; - let variables = Vec::new(); - let input = |index| Expr::Reference(Reference::Input(index)); let expression = Expr::Binary { op: BinaryOp::Subtract, left: Box::new(input(0)), @@ -186,30 +235,100 @@ mod tests { }; assert_eq!( - render(&expression, &inputs, &variables, Dialect::C).unwrap(), + render(&expression, &inputs(), &[], Dialect::C), "x - (y - z)" ); assert_eq!( - render(&expression, &inputs, &variables, Dialect::Cpp).unwrap(), + render(&expression, &inputs(), &[], Dialect::Cpp), "x - (y - z)" ); } #[test] - fn selects_cpp_math_namespace() { + fn preserves_parentheses_in_nested_unary_and_binary_expressions() { + let expression = Expr::Binary { + op: BinaryOp::Divide, + left: Box::new(Expr::Unary { + op: UnaryOp::Minus, + operand: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + }), + right: Box::new(Expr::Binary { + op: BinaryOp::Multiply, + left: Box::new(input(2)), + right: Box::new(Expr::Binary { + op: BinaryOp::Subtract, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + }), + }; + + assert_eq!( + render(&expression, &inputs(), &[], Dialect::C), + "-(x + y) / (z * (x - y))" + ); + } + + #[test] + fn unary_plus_preserves_the_operand_precedence() { + let expression = Expr::Binary { + op: BinaryOp::Multiply, + left: Box::new(Expr::Unary { + op: UnaryOp::Plus, + operand: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + }), + right: Box::new(input(2)), + }; + + assert_eq!( + render(&expression, &inputs(), &[], Dialect::C), + "(x + y) * z" + ); + } + + #[test] + fn renders_power_and_calls_with_dialect_local_math_names() { let expression = Expr::Call { - function: MathFunction::Sqrt, - args: vec![Expr::Reference(Reference::Input(0))], + function: MathFunction::Min, + args: vec![ + Expr::Binary { + op: BinaryOp::Power, + left: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + right: Box::new(Expr::Unary { + op: UnaryOp::Minus, + operand: Box::new(input(2)), + }), + }, + Expr::Call { + function: MathFunction::Sqrt, + args: vec![Expr::Binary { + op: BinaryOp::Multiply, + left: Box::new(input(0)), + right: Box::new(input(1)), + }], + }, + ], }; - let inputs = vec!["x".into()]; assert_eq!( - render(&expression, &inputs, &[], Dialect::C).unwrap(), - "sqrt(x)" + render(&expression, &inputs(), &[], Dialect::C), + "fmin(pow(x + y, -z), sqrt(x * y))" ); assert_eq!( - render(&expression, &inputs, &[], Dialect::Cpp).unwrap(), - "std::sqrt(x)" + render(&expression, &inputs(), &[], Dialect::Cpp), + "std::fmin(std::pow(x + y, -z), std::sqrt(x * y))" ); } } diff --git a/codegen/src/targets/native.rs b/codegen/src/targets/native.rs index dbf9007..9a259f9 100644 --- a/codegen/src/targets/native.rs +++ b/codegen/src/targets/native.rs @@ -228,7 +228,7 @@ fn render_function(function: &CompiledFunction, cpp: bool) -> Result { variables.push_str(&format!( " const double {} = {};\n", variable.name, - expression(&variable.expression, function, cpp)? + expression(&variable.expression, function, cpp) )); } let returned = match &function.core.output { @@ -241,7 +241,7 @@ fn render_function(function: &CompiledFunction, cpp: bool) -> Result { .expression, function, cpp, - )?, + ), Output::Scalar => output_name.clone(), Output::Struct(fields) if cpp => format!("{result}{{{}}}", fields.join(", ")), Output::Struct(fields) => format!("{result}{{{}}}", fields.join(", ")), @@ -401,7 +401,7 @@ fn expression( expression: &crate::semantic::Expr, function: &CompiledFunction, cpp: bool, -) -> Result { +) -> String { c_expression::render( expression, &function.core.inputs, diff --git a/codegen/src/targets/py/c.rs b/codegen/src/targets/py/c.rs index f68ed59..a58a0c7 100644 --- a/codegen/src/targets/py/c.rs +++ b/codegen/src/targets/py/c.rs @@ -69,7 +69,7 @@ fn ufunc(function: &CompiledFunction) -> Result { inputs, &function.ir.variables, Dialect::C, - )? + ) )); } let values = match &function.core.output { From 2963b5bac09d89e23abaac72c6e9a2aeba620291 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 22:36:00 +0300 Subject: [PATCH 04/14] refactor(codegen): structure native generation --- codegen/src/targets/c_expression.rs | 14 +- codegen/src/targets/native.rs | 673 ++++++++++++++++------------ 2 files changed, 401 insertions(+), 286 deletions(-) diff --git a/codegen/src/targets/c_expression.rs b/codegen/src/targets/c_expression.rs index 895d96d..f64a1b5 100644 --- a/codegen/src/targets/c_expression.rs +++ b/codegen/src/targets/c_expression.rs @@ -9,18 +9,26 @@ pub(super) enum Dialect { } pub(super) fn render( - expression: &Expr, + value: &Expr, inputs: &[String], variables: &[Variable], dialect: Dialect, ) -> String { + expression(value, inputs, variables, dialect).to_string() +} + +pub(super) fn expression<'a>( + value: &'a Expr, + inputs: &'a [String], + variables: &'a [Variable], + dialect: Dialect, +) -> impl fmt::Display + 'a { Expression { - expression, + expression: value, inputs, variables, dialect, } - .to_string() } pub(super) fn float_literal(lexeme: &str) -> String { diff --git a/codegen/src/targets/native.rs b/codegen/src/targets/native.rs index 9a259f9..586f19e 100644 --- a/codegen/src/targets/native.rs +++ b/codegen/src/targets/native.rs @@ -3,7 +3,10 @@ use std::{collections::BTreeSet, path::PathBuf}; use anyhow::Result; use convert_case::{Boundary, Case, Casing}; -use crate::model::{CompiledFunction, Function, Output, Parameter, Scope, Source}; +use crate::{ + model::{CompiledFunction, Function, Output, Parameter, Scope, Source}, + render::{Render, Writer}, +}; use super::{ c_expression::{self, Dialect}, @@ -26,10 +29,10 @@ pub(super) fn render(functions: &[CompiledFunction]) -> Result { let mut cpp_modules = Vec::new(); let mut c_tests = Vec::new(); let mut cpp_tests = Vec::new(); - let mut c_includes = String::new(); + let mut c_includes = Writer::new(); let mut module_paths = vec!["cpp/ptfkit.cppm".to_owned()]; for (slug, functions) in group_by_source(functions) { - c_includes.push_str(&format!("#include \n")); + c_includes.line(format_args!("#include ")); c_headers.push(file( format!("ptfkit/{slug}.h"), c_header(slug, &functions)?, @@ -39,34 +42,41 @@ pub(super) fn render(functions: &[CompiledFunction]) -> Result { c_tests.push(file(format!("{slug}.c"), c_test(slug, &functions)?)); cpp_tests.push(file(format!("{slug}.cpp"), cpp_test(slug, &functions)?)); } - c_headers.push(file( - "ptfkit/ptfkit.h", - format!( - "{HEADER}\n#ifndef PTFKIT_PTFKIT_H\n#define PTFKIT_PTFKIT_H\n\n{c_includes}\n#endif\n" - ), - )); - let exports = module_paths - .iter() - .skip(1) - .map(|path| path.trim_start_matches("cpp/").trim_end_matches(".cppm")) - .map(|slug| format!("export import ptfkit.{slug};\n")) - .collect::(); - cpp_modules.push(file( - "ptfkit.cppm", - format!("{HEADER}\nexport module ptfkit;\n\n{exports}"), - )); - let module_paths = module_paths - .iter() - .map(|path| format!(" \"${{CMAKE_CURRENT_LIST_DIR}}/../{path}\"")) - .collect::>() - .join("\n"); + let mut umbrella = Writer::new(); + umbrella.write(HEADER); + umbrella.blank_line(); + umbrella.line("#ifndef PTFKIT_PTFKIT_H"); + umbrella.line("#define PTFKIT_PTFKIT_H"); + umbrella.blank_line(); + umbrella.write(c_includes.into_string()); + umbrella.blank_line(); + umbrella.line("#endif"); + c_headers.push(file("ptfkit/ptfkit.h", umbrella.into_string())); + + let mut root_module = Writer::new(); + root_module.write(HEADER); + root_module.blank_line(); + root_module.line("export module ptfkit;"); + root_module.blank_line(); + for path in module_paths.iter().skip(1) { + let slug = path.trim_start_matches("cpp/").trim_end_matches(".cppm"); + root_module.line(format_args!("export import ptfkit.{slug};")); + } + cpp_modules.push(file("ptfkit.cppm", root_module.into_string())); + + let mut cmake = Writer::new(); + cmake.write(CMAKE_HEADER); + cmake.line("set(PTFKIT_CPP_MODULES"); + cmake.indented(|writer| { + for path in &module_paths { + writer.line(format_args!("\"${{CMAKE_CURRENT_LIST_DIR}}/../{path}\"")); + } + }); + cmake.line(")"); Ok(OutputFiles { c_headers, cpp_modules, - cpp_cmake: vec![file( - "ptfkitModules.cmake", - format!("{CMAKE_HEADER}set(PTFKIT_CPP_MODULES\n{module_paths}\n)\n"), - )], + cpp_cmake: vec![file("ptfkitModules.cmake", cmake.into_string())], c_tests, cpp_tests, }) @@ -84,17 +94,20 @@ pub(super) fn c_result_name(schema: &str) -> String { fn c_header(slug: &str, functions: &[&CompiledFunction]) -> Result { let guard = format!("PTFKIT_{}_H", slug.to_ascii_uppercase()); - let mut body = String::new(); + let mut writer = Writer::new(); + writer.write(HEADER); + writer.blank_line(); + writer.line(format_args!("#ifndef {guard}")); + writer.line(format_args!("#define {guard}")); + writer.blank_line(); if requires_math(functions) { - body.push_str("#include \n"); + writer.line("#include "); + writer.blank_line(); } let first = functions .first() .expect("generated source contains at least one function"); - body.push_str(&format!( - "\n{}", - source_comment(&first.entry.spec.source, &first.entry.spec.scope) - )); + source_comment(&first.entry.spec.source, &first.entry.spec.scope).render(&mut writer); let mut schemas = BTreeSet::new(); for function in functions { let spec = &function.entry.spec.functions[function.function_index]; @@ -103,168 +116,261 @@ fn c_header(slug: &str, functions: &[&CompiledFunction]) -> Result { .result_class() .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; if schemas.insert(schema) { - body.push_str(&format!( - "\ntypedef struct {{\n{} }} {};\n", - fields - .iter() - .map(|field| { - let parameter = spec - .outputs - .fields() - .iter() - .find(|parameter| parameter.name == *field) - .expect("core output field matches specification"); - format!(" {}\n double {field};\n", field_comment(parameter)) - }) - .collect::(), - c_result_name(schema) - )); + writer.blank_line(); + render_struct( + &mut writer, + fields, + spec, + &c_result_name(schema), + NativeDialect::C, + ); } } } for function in functions { - body.push_str(&format!("\n{}", c_function(function)?)); + writer.blank_line(); + NativeFunction::c(function)?.render(&mut writer); } - Ok(format!( - "{HEADER}\n#ifndef {guard}\n#define {guard}\n\n{body}\n#endif\n" - )) + writer.blank_line(); + writer.line("#endif"); + Ok(writer.into_string()) } fn cpp_module(slug: &str, functions: &[&CompiledFunction]) -> Result { - let mut body = String::new(); + let mut writer = Writer::new(); + writer.write(HEADER); + writer.blank_line(); if requires_math(functions) { - body.push_str("module;\n#include \n\n"); + writer.line("module;"); + writer.line("#include "); + writer.blank_line(); } - body.push_str(&format!("export module ptfkit.{slug};\n\n")); + writer.line(format_args!("export module ptfkit.{slug};")); + writer.blank_line(); let first = functions .first() .expect("generated source contains at least one function"); - body.push_str(&source_comment( - &first.entry.spec.source, - &first.entry.spec.scope, - )); - body.push('\n'); - body.push_str(&format!("export namespace ptfkit::{slug} {{\n")); - let mut schemas = BTreeSet::new(); - for function in functions { - let spec = &function.entry.spec.functions[function.function_index]; - if let Output::Struct(fields) = &function.core.output { - let result = spec - .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; - if schemas.insert(result) { - body.push_str(&format!( - "\nstruct {} {{\n{} }};\n", - result, - fields - .iter() - .map(|field| { - let parameter = spec - .outputs - .fields() - .iter() - .find(|parameter| parameter.name == *field) - .expect("core output field matches specification"); - format!(" {}\n double {field};\n", field_comment(parameter)) - }) - .collect::() - )); + source_comment(&first.entry.spec.source, &first.entry.spec.scope).render(&mut writer); + writer.blank_line(); + let native_functions = functions + .iter() + .map(|function| NativeFunction::cpp(function)) + .collect::>>()?; + writer.line(format_args!("export namespace ptfkit::{slug} {{")); + writer.indented(|writer| { + let mut schemas = BTreeSet::new(); + for function in functions { + let spec = &function.entry.spec.functions[function.function_index]; + if let Output::Struct(fields) = &function.core.output { + let result = spec + .result_class() + .expect("record output has a result class"); + if schemas.insert(result) { + writer.blank_line(); + render_struct(writer, fields, spec, result, NativeDialect::Cpp); + } } } - } - for function in functions { - body.push_str(&format!("\n{}", cpp_function(function)?)); - } - body.push_str(&format!("\n}} // namespace ptfkit::{slug}\n")); - Ok(format!("{HEADER}\n{body}")) + for function in &native_functions { + writer.blank_line(); + function.render(writer); + } + }); + writer.blank_line(); + writer.line(format_args!("}} // namespace ptfkit::{slug}")); + Ok(writer.into_string()) } -fn c_function(function: &CompiledFunction) -> Result { - render_function(function, false) +#[derive(Clone, Copy)] +enum NativeDialect { + C, + Cpp, } -fn cpp_function(function: &CompiledFunction) -> Result { - render_function(function, true) + +struct NativeFunction<'a> { + function: &'a CompiledFunction, + result: String, + dialect: NativeDialect, + terminal: bool, } -fn render_function(function: &CompiledFunction, cpp: bool) -> Result { - let spec = &function.entry.spec.functions[function.function_index]; - let result = match function.core.output { - Output::Scalar => "double".to_owned(), - Output::Struct(_) if cpp => spec - .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))? - .to_owned(), - Output::Struct(_) => c_result_name( - spec.result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?, - ), - }; - let prefix = if cpp { - "[[nodiscard]]\ninline" - } else { - "static inline" - }; - let inputs = function - .core - .inputs - .iter() - .map(|input| format!("double {input}")) - .collect::>() - .join(", "); - let mut variables = String::new(); - let output_name = &spec.outputs.fields()[0].name; - let terminal = matches!(function.core.output, Output::Scalar) - && function - .ir - .variables - .last() - .is_some_and(|v| v.name == *output_name); - for variable in function - .ir - .variables - .iter() - .take(function.ir.variables.len() - usize::from(terminal)) - { - variables.push_str(&format!( - " const double {} = {};\n", - variable.name, - expression(&variable.expression, function, cpp) - )); +impl<'a> NativeFunction<'a> { + fn c(function: &'a CompiledFunction) -> Result { + Self::new(function, NativeDialect::C) } - let returned = match &function.core.output { - Output::Scalar if terminal => expression( - &function + + fn cpp(function: &'a CompiledFunction) -> Result { + Self::new(function, NativeDialect::Cpp) + } + + fn new(function: &'a CompiledFunction, dialect: NativeDialect) -> Result { + let spec = &function.entry.spec.functions[function.function_index]; + let result = match function.core.output { + Output::Scalar => "double".to_owned(), + Output::Struct(_) if matches!(dialect, NativeDialect::Cpp) => spec + .result_class() + .ok_or_else(|| anyhow::anyhow!("record output has no result class"))? + .to_owned(), + Output::Struct(_) => c_result_name( + spec.result_class() + .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?, + ), + }; + let output_name = &spec.outputs.fields()[0].name; + let terminal = matches!(function.core.output, Output::Scalar) + && function .ir .variables .last() - .expect("terminal variable") - .expression, + .is_some_and(|variable| variable.name == *output_name); + Ok(Self { function, - cpp, - ), - Output::Scalar => output_name.clone(), - Output::Struct(fields) if cpp => format!("{result}{{{}}}", fields.join(", ")), - Output::Struct(fields) => format!("{result}{{{}}}", fields.join(", ")), - }; - let return_statement = match &function.core.output { - Output::Struct(fields) if !cpp => format!( - "#ifdef __cplusplus\n return {result}{{{}}};\n#else\n return ({result}) {{\n{} }};\n#endif", - fields.join(", "), - fields + result, + dialect, + terminal, + }) + } +} + +impl Render for NativeFunction<'_> { + fn render(&self, writer: &mut Writer) { + let spec = &self.function.entry.spec.functions[self.function.function_index]; + function_comment(spec).render(writer); + if matches!(self.dialect, NativeDialect::Cpp) { + writer.line("[[nodiscard]]"); + writer.write("inline "); + } else { + writer.write("static inline "); + } + writer.write(&self.result); + writer.write(" "); + writer.write(&self.function.core.name); + writer.write("("); + for (index, input) in self.function.core.inputs.iter().enumerate() { + if index > 0 { + writer.write(", "); + } + writer.write("double "); + writer.write(input); + } + writer.line(") {"); + writer.indented(|writer| { + for variable in self + .function + .ir + .variables .iter() - .map(|field| format!(" .{field} = {field},\n")) - .collect::() - ), - _ => format!("return {returned};"), - }; - Ok(format!( - "{}{prefix} {result} {}({inputs}) {{\n{variables} {return_statement}\n}}\n", - function_comment(spec), - function.core.name - )) + .take(self.function.ir.variables.len() - usize::from(self.terminal)) + { + writer.write(format_args!("const double {} = ", variable.name)); + writer.write(c_expression::expression( + &variable.expression, + &self.function.core.inputs, + &self.function.ir.variables, + self.expression_dialect(), + )); + writer.line(";"); + } + self.render_return(writer); + }); + writer.line("}"); + } } -fn source_comment(source: &Source, scope: &Scope) -> String { +impl NativeFunction<'_> { + fn expression_dialect(&self) -> Dialect { + match self.dialect { + NativeDialect::C => Dialect::C, + NativeDialect::Cpp => Dialect::Cpp, + } + } + + fn render_return(&self, writer: &mut Writer) { + let output_name = &self.function.entry.spec.functions[self.function.function_index] + .outputs + .fields()[0] + .name; + match &self.function.core.output { + Output::Scalar if self.terminal => { + writer.write("return "); + writer.write(c_expression::expression( + &self + .function + .ir + .variables + .last() + .expect("terminal variable") + .expression, + &self.function.core.inputs, + &self.function.ir.variables, + self.expression_dialect(), + )); + writer.line(";"); + } + Output::Scalar => writer.line(format_args!("return {output_name};")), + Output::Struct(fields) if matches!(self.dialect, NativeDialect::C) => { + writer.line("#ifdef __cplusplus"); + writer.write(format_args!("return {}{{", self.result)); + render_values(writer, fields); + writer.line("};"); + writer.line("#else"); + writer.line(format_args!("return ({}) {{", self.result)); + writer.indented(|writer| { + for field in fields { + writer.line(format_args!(".{field} = {field},")); + } + }); + writer.line("};"); + writer.line("#endif"); + } + Output::Struct(fields) => { + writer.write(format_args!("return {}{{", self.result)); + render_values(writer, fields); + writer.line("};"); + } + } + } +} + +fn render_values(writer: &mut Writer, values: &[String]) { + for (index, value) in values.iter().enumerate() { + if index > 0 { + writer.write(", "); + } + writer.write(value); + } +} + +fn render_struct( + writer: &mut Writer, + fields: &[String], + spec: &Function, + name: &str, + dialect: NativeDialect, +) { + match dialect { + NativeDialect::C => writer.line("typedef struct {"), + NativeDialect::Cpp => writer.line(format_args!("struct {name} {{")), + } + writer.indented(|writer| { + for field in fields { + let parameter = spec + .outputs + .fields() + .iter() + .find(|parameter| parameter.name == *field) + .expect("core output field matches specification"); + field_comment(parameter).render(writer); + writer.line(format_args!("double {field};")); + } + }); + match dialect { + NativeDialect::C => writer.line(format_args!("}} {name};")), + NativeDialect::Cpp => writer.line("};"), + } +} + +fn source_comment(source: &Source, scope: &Scope) -> Comment { let mut lines = vec![ format!("@brief {}", source.summary), String::new(), @@ -288,10 +394,10 @@ fn source_comment(source: &Source, scope: &Scope) -> String { dataset.clone(), ]); } - comment(lines) + Comment(lines) } -fn function_comment(function: &Function) -> String { +fn function_comment(function: &Function) -> Comment { let mut lines = vec![format!("@brief {}", function.public_api.summary)]; lines.extend(function.inputs.iter().map(|parameter| { format!( @@ -343,37 +449,30 @@ fn function_comment(function: &Function) -> String { .iter() .map(|warning| format!("@warning {warning}")), ); - comment(lines) + Comment(lines) } -fn field_comment(parameter: &Parameter) -> String { - comment([format!( +fn field_comment(parameter: &Parameter) -> Comment { + Comment(vec![format!( "@brief {}", documentation::parameter_details(parameter) )]) - .trim_end() - .to_owned() } -fn comment(lines: impl IntoIterator) -> String { - let lines = lines - .into_iter() - .flat_map(|line| wrap_comment_line(&line)) - .map(|line| line.replace("*/", "* /")) - .collect::>(); - format!( - "/**\n{} */\n", - lines - .into_iter() - .map(|line| { - if line.is_empty() { - " *\n".to_owned() - } else { - format!(" * {line}\n") - } - }) - .collect::() - ) +struct Comment(Vec); + +impl Render for Comment { + fn render(&self, writer: &mut Writer) { + writer.line("/**"); + for line in self.0.iter().flat_map(|line| wrap_comment_line(line)) { + if line.is_empty() { + writer.line(" *"); + } else { + writer.line(format_args!(" * {}", line.replace("*/", "* /"))); + } + } + writer.line(" */"); + } } fn wrap_comment_line(line: &str) -> Vec { @@ -397,19 +496,6 @@ fn wrap_comment_line(line: &str) -> Vec { lines } -fn expression( - expression: &crate::semantic::Expr, - function: &CompiledFunction, - cpp: bool, -) -> String { - c_expression::render( - expression, - &function.core.inputs, - &function.ir.variables, - if cpp { Dialect::Cpp } else { Dialect::C }, - ) -} - fn requires_math(functions: &[&CompiledFunction]) -> bool { functions.iter().any(|function| { function @@ -428,102 +514,123 @@ fn cpp_test(slug: &str, functions: &[&CompiledFunction]) -> Result { } fn c_compatibility_test(slug: &str, functions: &[&CompiledFunction]) -> Result { - let includes = ""; - let mut tests = String::new(); - for function in functions { - let spec = &function.entry.spec.functions[function.function_index]; - for case in &function.golden_tests { - let values = case - .inputs - .iter() - .map(|value| c_expression::test_float_literal(*value)) - .collect::>(); - let call = format!("{}({})", function.core.name, values.join(", ")); - let result_declaration = if matches!(function.core.output, Output::Scalar) { - "const double" - } else { - "const ptfkit_result_placeholder" - }; - let result_declaration = if result_declaration == "const ptfkit_result_placeholder" { - let name = spec - .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; - format!("const {}", c_result_name(name)) - } else { - result_declaration.to_owned() - }; - tests.push_str(&format!( - " {{\n {result_declaration} result = {call};\n" - )); - for (field, expected) in spec.outputs.fields().iter().zip(&case.expected) { - let actual = if matches!(function.core.output, Output::Scalar) { - "result".to_owned() - } else { - format!("result.{}", field.name) - }; - tests.push_str(&format!( - " assert_close_enough({actual}, {}, {}, {});\n", - c_expression::test_float_literal(*expected), - c_expression::test_float_literal(case.atol), - c_expression::test_float_literal(case.rtol), - )); + let mut writer = Writer::new(); + writer.write(HEADER); + writer.blank_line(); + writer.line(format_args!("#include ")); + writer.line("#include \"close_enough.h\""); + writer.blank_line(); + writer.line("int main() {"); + writer.indented(|writer| { + for function in functions { + let spec = &function.entry.spec.functions[function.function_index]; + for case in &function.golden_tests { + writer.line("{"); + writer.indented(|writer| { + if matches!(function.core.output, Output::Scalar) { + writer.write("const double"); + } else { + let name = spec + .result_class() + .expect("record output has a result class"); + writer.write(format_args!("const {}", c_result_name(name))); + } + writer.write(" result = "); + writer.write(&function.core.name); + writer.write("("); + render_literals(writer, &case.inputs); + writer.line(");"); + for (field, expected) in spec.outputs.fields().iter().zip(&case.expected) { + writer.write("assert_close_enough("); + if matches!(function.core.output, Output::Scalar) { + writer.write("result"); + } else { + writer.write(format_args!("result.{}", field.name)); + } + writer.write(", "); + writer.write(c_expression::test_float_literal(*expected)); + writer.write(", "); + writer.write(c_expression::test_float_literal(case.atol)); + writer.write(", "); + writer.write(c_expression::test_float_literal(case.rtol)); + writer.line(");"); + } + }); + writer.line("}"); } - tests.push_str(" }\n"); } - } - Ok(format!( - "{HEADER}\n#include \n#include \"close_enough.h\"\n{includes}\nint main() {{\n{tests} return 0;\n}}\n" - )) + }); + writer.line(" return 0;"); + writer.line("}"); + Ok(writer.into_string()) } fn module_test(slug: &str, functions: &[&CompiledFunction]) -> Result { let type_traits = functions .iter() .any(|function| matches!(function.core.output, Output::Struct(_))); - let includes = if type_traits { - "#include \n" - } else { - "" - }; - let mut tests = String::new(); + let mut writer = Writer::new(); + writer.write(HEADER); + writer.blank_line(); + writer.line("#ifdef IMPORT_UMBRELLA"); + writer.line("import ptfkit;"); + writer.line("#else"); + writer.line(format_args!("import ptfkit.{slug};")); + writer.line("#endif"); + writer.blank_line(); + writer.line("#include \"close_enough.h\""); + if type_traits { + writer.line("#include "); + } + writer.blank_line(); + writer.line("int main() {"); + writer.indented(|writer| { for function in functions { let spec = &function.entry.spec.functions[function.function_index]; for case in &function.golden_tests { - let values = case - .inputs - .iter() - .map(|value| c_expression::test_float_literal(*value)) - .collect::>(); - tests.push_str(&format!( - " {{\n const auto result = ptfkit::{slug}::{}({});\n", - function.core.name, - values.join(", ") - )); + writer.line("{"); + writer.indented(|writer| { + writer.write(format_args!("const auto result = ptfkit::{slug}::{}(", function.core.name)); + render_literals(writer, &case.inputs); + writer.line(");"); if matches!(function.core.output, Output::Struct(_)) { let result = spec .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; - tests.push_str(&format!(" static_assert(std::is_same_v, ptfkit::{slug}::{result}>);\n")); + .expect("record output has a result class"); + writer.line(format_args!("static_assert(std::is_same_v, ptfkit::{slug}::{result}>);")); } for (field, expected) in spec.outputs.fields().iter().zip(&case.expected) { - let actual = if matches!(function.core.output, Output::Scalar) { - "result".to_owned() + writer.write("assert_close_enough("); + if matches!(function.core.output, Output::Scalar) { + writer.write("result"); } else { - format!("result.{}", field.name) - }; - tests.push_str(&format!( - " assert_close_enough({actual}, {}, {}, {});\n", - c_expression::test_float_literal(*expected), - c_expression::test_float_literal(case.atol), - c_expression::test_float_literal(case.rtol), - )); + writer.write(format_args!("result.{}", field.name)); + } + writer.write(", "); + writer.write(c_expression::test_float_literal(*expected)); + writer.write(", "); + writer.write(c_expression::test_float_literal(case.atol)); + writer.write(", "); + writer.write(c_expression::test_float_literal(case.rtol)); + writer.line(");"); } - tests.push_str(" }\n"); + }); + writer.line("}"); + } + } + }); + writer.line(" return 0;"); + writer.line("}"); + Ok(writer.into_string()) +} + +fn render_literals(writer: &mut Writer, values: &[f64]) { + for (index, value) in values.iter().enumerate() { + if index > 0 { + writer.write(", "); } + writer.write(c_expression::test_float_literal(*value)); } - Ok(format!( - "{HEADER}\n#ifdef IMPORT_UMBRELLA\nimport ptfkit;\n#else\nimport ptfkit.{slug};\n#endif\n\n#include \"close_enough.h\"\n{includes}\nint main() {{\n{tests} return 0;\n}}\n" - )) } #[cfg(test)] From 57673a06b063ac7326df72c0d9e4d29b0e6b5a66 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 22:40:42 +0300 Subject: [PATCH 05/14] refactor(codegen): share documentation semantics --- codegen/src/documentation.rs | 193 +++++++++++++++++++++++++++++++++++ codegen/src/main.rs | 1 + codegen/src/targets/rs.rs | 88 +++++++++------- 3 files changed, 243 insertions(+), 39 deletions(-) create mode 100644 codegen/src/documentation.rs diff --git a/codegen/src/documentation.rs b/codegen/src/documentation.rs new file mode 100644 index 0000000..96616ae --- /dev/null +++ b/codegen/src/documentation.rs @@ -0,0 +1,193 @@ +//! Semantic documentation assembled from validated specifications. +//! +//! Targets choose their own section ordering and markup. This module only +//! describes the information they have available to render. + +use crate::model::{Function, Outputs, Parameter, Scope, Source}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SourceDocument<'a> { + pub(crate) summary: &'a str, + pub(crate) reference: Reference<'a>, + pub(crate) territory: Option<&'a str>, + pub(crate) dataset: Option<&'a str>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Reference<'a> { + pub(crate) citation: &'a str, + pub(crate) doi: Option>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Doi<'a> { + pub(crate) identifier: &'a str, + pub(crate) url: &'a str, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct FunctionDocument<'a> { + pub(crate) summary: &'a str, + pub(crate) parameters: &'a [Parameter], + pub(crate) returns: Returns<'a>, + pub(crate) territory: Option<&'a str>, + pub(crate) models: Models<'a>, + pub(crate) remarks: Remarks<'a>, + pub(crate) notes: &'a [String], + pub(crate) warnings: &'a [String], +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Returns<'a> { + Scalar(&'a Parameter), + Record { + name: &'a str, + fields: &'a [Parameter], + }, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Models<'a> { + pub(crate) h_theta: Option<&'a str>, + pub(crate) k_h: Option<&'a str>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Remarks<'a> { + pub(crate) prediction_target: &'a str, +} + +pub(crate) fn for_source<'a>(source: &'a Source, scope: &'a Scope) -> SourceDocument<'a> { + SourceDocument { + summary: &source.summary, + reference: Reference { + citation: &source.citation_apa, + doi: source.doi.as_ref().map(|doi| Doi { + identifier: &doi.identifier, + url: &doi.url, + }), + }, + territory: scope.territory.as_deref(), + dataset: scope.dataset.as_deref(), + } +} + +pub(crate) fn for_function(function: &Function) -> FunctionDocument<'_> { + FunctionDocument { + summary: &function.public_api.summary, + parameters: &function.inputs, + returns: match &function.outputs { + Outputs::Scalar { field } => Returns::Scalar(field), + Outputs::Record { name, fields } => Returns::Record { name, fields }, + }, + territory: function.scope.territory.as_deref(), + models: Models { + h_theta: function.scope.models.h_theta.as_deref(), + k_h: function.scope.models.k_h.as_deref(), + }, + remarks: Remarks { + prediction_target: &function.scope.prediction_target, + }, + notes: &function.documentation.notes, + warnings: &function.documentation.warnings, + } +} + +#[cfg(test)] +mod tests { + use crate::model::{ + Documentation, Function, FunctionScope, Models, Outputs, Parameter, PublicApi, + }; + + use super::{Returns, for_function}; + + fn parameter(name: &str) -> Parameter { + Parameter { + name: name.into(), + unit: "cm^3/cm^3".into(), + domain: None, + description: format!("{name} description."), + } + } + + #[test] + fn preserves_empty_optional_documentation_sections() { + let function = Function { + name: "calc_ptf_test".into(), + status: "draft".into(), + public_api: PublicApi { + name: "calc_ptf_test".into(), + result_class: None, + summary: "Estimate a test property.".into(), + }, + scope: FunctionScope { + territory: None, + prediction_target: "Test property.".into(), + models: Models::default(), + }, + inputs: Vec::new(), + outputs: Outputs::Scalar { + field: parameter("result"), + }, + documentation: Documentation::default(), + implementation: None, + golden_tests: Vec::new(), + }; + + let document = for_function(&function); + assert!(document.territory.is_none()); + assert!(document.models.h_theta.is_none()); + assert!(document.models.k_h.is_none()); + assert!(document.notes.is_empty()); + assert!(document.warnings.is_empty()); + assert!(matches!(document.returns, Returns::Scalar(_))); + } + + #[test] + fn retains_each_record_output_field() { + let function = Function { + name: "calc_ptf_test".into(), + status: "draft".into(), + public_api: PublicApi { + name: "calc_ptf_test".into(), + result_class: Some("TestResult".into()), + summary: "Estimate test properties.".into(), + }, + scope: FunctionScope { + territory: Some("Test territory.".into()), + prediction_target: "Test properties.".into(), + models: Models { + h_theta: Some("Retention model.".into()), + k_h: Some("Conductivity model.".into()), + }, + }, + inputs: vec![parameter("sand")], + outputs: Outputs::Record { + name: "TestResult".into(), + fields: vec![parameter("theta_33"), parameter("theta_1500")], + }, + documentation: Documentation { + notes: vec!["A note.".into()], + warnings: vec!["A warning.".into()], + }, + implementation: None, + golden_tests: Vec::new(), + }; + + let document = for_function(&function); + let Returns::Record { name, fields } = document.returns else { + panic!("record outputs must retain their shape"); + }; + assert_eq!(name, "TestResult"); + assert_eq!( + fields + .iter() + .map(|field| field.name.as_str()) + .collect::>(), + ["theta_33", "theta_1500"] + ); + assert_eq!(document.models.h_theta, Some("Retention model.")); + assert_eq!(document.models.k_h, Some("Conductivity model.")); + assert_eq!(document.remarks.prediction_target, "Test properties."); + } +} diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 6febf84..0570255 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -5,6 +5,7 @@ use std::{path::Path, process::ExitCode}; use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; +mod documentation; mod formula; mod model; mod render; diff --git a/codegen/src/targets/rs.rs b/codegen/src/targets/rs.rs index 85add22..c961b1a 100644 --- a/codegen/src/targets/rs.rs +++ b/codegen/src/targets/rs.rs @@ -5,7 +5,8 @@ use proc_macro2::{Ident, Literal, TokenStream, TokenTree}; use quote::{format_ident, quote}; use crate::{ - model::{CompiledFunction, Function, Output, Parameter, Scope, Source}, + documentation::{self as docs, FunctionDocument, Returns, SourceDocument}, + model::{CompiledFunction, Output}, semantic::{self, BinaryOp, Expr, MathFunction, Number, Reference, UnaryOp}, }; @@ -20,7 +21,10 @@ pub(crate) fn render(functions: &[CompiledFunction]) -> Result 1; let mut defined_result_classes = BTreeSet::new(); let definitions = functions @@ -49,7 +53,7 @@ fn module_tokens( ) -> Result { let function = &resolved.core; let specification = &resolved.entry.spec.functions[resolved.function_index]; - let function_docs = function_doc_tokens(specification); + let function_docs = function_doc_tokens(docs::for_function(specification)); let name = format_ident!("{}", function.name); let inputs = function .inputs @@ -177,75 +181,65 @@ fn output_tokens( } } -fn module_doc_tokens(source: &Source, scope: &Scope) -> TokenStream { +fn module_doc_tokens(document: SourceDocument<'_>) -> TokenStream { let mut lines = vec![ - source.summary.clone(), + document.summary.into(), String::new(), "# Reference".into(), String::new(), - source.citation_apa.clone(), + document.reference.citation.into(), ]; - if let Some(doi) = &source.doi { + if let Some(doi) = document.reference.doi { lines.push(format!("DOI: {} ({})", doi.identifier, doi.url)); } - if let Some(territory) = &scope.territory { + if let Some(territory) = document.territory { lines.extend([ String::new(), "# Territory".into(), String::new(), - territory.clone(), + territory.into(), ]); } - if let Some(dataset) = &scope.dataset { + if let Some(dataset) = document.dataset { lines.extend([ String::new(), "# Dataset".into(), String::new(), - dataset.clone(), + dataset.into(), ]); } inner_doc_tokens(lines) } -fn function_doc_tokens(function: &Function) -> TokenStream { +fn function_doc_tokens(document: FunctionDocument<'_>) -> TokenStream { let mut lines = vec![ - function.public_api.summary.clone(), + document.summary.into(), String::new(), "# Arguments".into(), String::new(), ]; lines.extend( - function - .inputs + document + .parameters .iter() .map(|parameter| format!("* {}", documentation::parameter_documentation(parameter))), ); lines.extend([String::new(), "# Returns".into(), String::new()]); - lines.extend(return_doc_lines( - function.result_class(), - function.outputs.fields(), - )); - if let Some(territory) = &function.scope.territory { + lines.extend(return_doc_lines(document.returns)); + if let Some(territory) = document.territory { lines.extend([ String::new(), "# Territory".into(), String::new(), - territory.clone(), + territory.into(), ]); } let models = [ - function - .scope + document .models .h_theta - .as_ref() .map(|model| format!("h(theta): {model}")), - function - .scope - .models - .k_h - .as_ref() - .map(|model| format!("k(h): {model}")), + document.models.k_h.map(|model| format!("k(h): {model}")), ] .into_iter() .flatten() @@ -258,20 +252,26 @@ fn function_doc_tokens(function: &Function) -> TokenStream { String::new(), "# Notes".into(), String::new(), - format!("Prediction target: {}", function.scope.prediction_target), + format!("Prediction target: {}", document.remarks.prediction_target), ]); - lines.extend(function.documentation.notes.iter().cloned()); - if !function.documentation.warnings.is_empty() { + lines.extend(document.notes.iter().cloned()); + if !document.warnings.is_empty() { lines.extend([String::new(), "# Warnings".into(), String::new()]); - lines.extend(function.documentation.warnings.iter().cloned()); + lines.extend(document.warnings.iter().cloned()); } doc_tokens(lines) } -fn return_doc_lines(result_class: Option<&str>, outputs: &[Parameter]) -> Vec { - match result_class { - Some(result_class) => vec![format!("A [`{result_class}`].")], - None => outputs +fn return_doc_lines(returns: Returns<'_>) -> Vec { + match returns { + Returns::Record { name, fields } => { + debug_assert!( + !fields.is_empty(), + "record outputs contain at least one field" + ); + vec![format!("A [`{name}`].")] + } + Returns::Scalar(output) => std::slice::from_ref(output) .iter() .map(|parameter| format!("* {}", documentation::parameter_documentation(parameter))) .collect(), @@ -570,6 +570,8 @@ fn render_tokens(module_docs: TokenStream, tokens: TokenStream) -> String { #[cfg(test)] mod tests { use super::*; + use crate::documentation::Returns; + use crate::model::Parameter; use crate::semantic::{BinaryOp, Expr, MathFunction, Reference}; fn number(value: f64) -> Expr { @@ -727,7 +729,15 @@ mod tests { #[test] fn record_return_documentation_links_to_the_result_type() { assert_eq!( - return_doc_lines(Some("Li2007PTFResult"), &[]), + return_doc_lines(Returns::Record { + name: "Li2007PTFResult", + fields: &[Parameter { + name: "theta_s".into(), + unit: "cm^3/cm^3".into(), + domain: None, + description: "Saturated water content.".into(), + }], + }), ["A [`Li2007PTFResult`]."] ); } From 52f8ef9b715329d817d6a20b4cd00c4485e925f4 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 22:49:15 +0300 Subject: [PATCH 06/14] refactor(codegen): migrate documentation renderers --- codegen/src/targets/c_documentation.rs | 56 +++++++++++-------- codegen/src/targets/cpp_documentation.rs | 60 +++++++++++++-------- codegen/src/targets/documentation.rs | 69 +++++++++++++----------- codegen/src/targets/native.rs | 49 +++++++++-------- codegen/src/targets/py/wrapper.rs | 67 ++++++++++++----------- 5 files changed, 172 insertions(+), 129 deletions(-) diff --git a/codegen/src/targets/c_documentation.rs b/codegen/src/targets/c_documentation.rs index 7604aec..45ba474 100644 --- a/codegen/src/targets/c_documentation.rs +++ b/codegen/src/targets/c_documentation.rs @@ -2,7 +2,10 @@ use std::{collections::BTreeSet, path::PathBuf}; use anyhow::Result; -use crate::model::{CompiledFunction, Output, Parameter}; +use crate::{ + documentation::{self as docs}, + model::{CompiledFunction, Output, Parameter}, +}; use super::{ GeneratedFile, @@ -56,7 +59,11 @@ fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> "- [``](headers/ptfkit.md) — Aggregates every ptfkit source header.\n", ); for (slug, functions) in sources { - let summary = &functions[0].entry.spec.source.summary; + let summary = docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope, + ) + .summary; text.push_str(&format!( "- [``](headers/{slug}.md) — {}\n", escape_text(summary) @@ -75,7 +82,13 @@ fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) for (slug, functions) in sources { text.push_str(&format!( "- [``]({slug}.md) — {}\n", - escape_text(&functions[0].entry.spec.source.summary) + escape_text( + docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope + ) + .summary, + ) )); } text @@ -85,28 +98,27 @@ fn header(slug: &str, functions: &[&CompiledFunction]) -> Result { let first = functions .first() .expect("compiled source contains at least one function"); - let source = &first.entry.spec.source; - let scope = &first.entry.spec.scope; + let source = docs::for_source(&first.entry.spec.source, &first.entry.spec.scope); let mut text = String::from(HEADER); text.push_str(&format!("# ``\n\n")); text.push_str(&format!("```c\n#include \n```\n\n")); - text.push_str(&format!("{}\n\n", escape_text(&source.summary))); + text.push_str(&format!("{}\n\n", escape_text(source.summary))); text.push_str("## Source\n\n"); - text.push_str(&escape_text(&source.citation_apa)); + text.push_str(&escape_text(source.reference.citation)); text.push_str("\n\n"); - if let Some(doi) = &source.doi { + if let Some(doi) = source.reference.doi { text.push_str(&format!( "[DOI: {}]({})\n\n", - escape_text(&doi.identifier), + escape_text(doi.identifier), doi.url )); } - if scope.territory.is_some() || scope.dataset.is_some() { + if source.territory.is_some() || source.dataset.is_some() { text.push_str("## Scope\n\n"); - if let Some(territory) = &scope.territory { + if let Some(territory) = source.territory { text.push_str(&format!("**Territory:** {}\n\n", escape_text(territory))); } - if let Some(dataset) = &scope.dataset { + if let Some(dataset) = source.dataset { text.push_str(&format!("**Dataset:** {}\n\n", escape_text(dataset))); } } @@ -158,14 +170,15 @@ fn structure(name: &str, fields: &[Parameter]) -> String { fn function_documentation(function: &CompiledFunction) -> Result { let spec = spec(function); + let document = docs::for_function(spec); let anchor = function_anchor(&function.core.name); let mut text = format!("### `{}` {{#{anchor}}}\n\n", function.core.name); - text.push_str(&format!("{}\n\n", escape_text(&spec.public_api.summary))); + text.push_str(&format!("{}\n\n", escape_text(document.summary))); text.push_str("```c\n"); text.push_str(&signature(function)?); text.push_str(";\n```\n\n"); text.push_str("#### Parameters\n\n| Name | Direction | Description |\n| --- | --- | --- |\n"); - for parameter in &spec.inputs { + for parameter in document.parameters { text.push_str(&format!( "| `{}` | in | {} |\n", parameter.name, @@ -173,22 +186,19 @@ fn function_documentation(function: &CompiledFunction) -> Result { )); } text.push_str("\n#### Returns\n\n"); - match &function.core.output { - Output::Scalar => text.push_str(&format!( - "{}\n\n", - parameter_details(&spec.outputs.fields()[0]) - )), - Output::Struct(_) => { + match document.returns { + docs::Returns::Scalar(field) => text.push_str(&format!("{}\n\n", parameter_details(field))), + docs::Returns::Record { .. } => { let name = spec .result_class() .expect("record output has a result class"); text.push_str(&format!("A `{}` value.\n\n", c_result_name(name))); } } - for note in &spec.documentation.notes { + for note in document.notes { admonition(&mut text, "note", note); } - for warning in &spec.documentation.warnings { + for warning in document.warnings { admonition(&mut text, "warning", warning); } Ok(text) @@ -203,7 +213,7 @@ fn functions_index(functions: &[(&str, &CompiledFunction)]) -> String { function.core.name, slug, function_anchor(&function.core.name), - escape_table(&spec(function).public_api.summary), + escape_table(docs::for_function(spec(function)).summary), )); } text diff --git a/codegen/src/targets/cpp_documentation.rs b/codegen/src/targets/cpp_documentation.rs index b8c6a6b..99137c8 100644 --- a/codegen/src/targets/cpp_documentation.rs +++ b/codegen/src/targets/cpp_documentation.rs @@ -2,7 +2,10 @@ use std::{collections::BTreeSet, path::PathBuf}; use anyhow::{Result, anyhow}; -use crate::model::{CompiledFunction, Output, Parameter}; +use crate::{ + documentation::{self as docs}, + model::{CompiledFunction, Output, Parameter}, +}; use super::{GeneratedFile, documentation::FRONTMATTER_HEADER, group_by_source}; @@ -42,7 +45,13 @@ fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> for (slug, functions) in sources { text.push_str(&format!( "- [`ptfkit.{slug}`](modules/{slug}.md) — {}\n", - escape_text(&functions[0].entry.spec.source.summary) + escape_text( + docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope + ) + .summary, + ) )); } text.push_str("\nSee the [function index](functions.md) for all public C++ functions.\n"); @@ -58,7 +67,13 @@ fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) for (slug, functions) in sources { text.push_str(&format!( "- [`ptfkit.{slug}`]({slug}.md) — {}\n", - escape_text(&functions[0].entry.spec.source.summary) + escape_text( + docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope + ) + .summary, + ) )); } text @@ -68,8 +83,7 @@ fn module(slug: &str, functions: &[&CompiledFunction]) -> Result { let first = functions .first() .expect("compiled source contains at least one function"); - let source = &first.entry.spec.source; - let scope = &first.entry.spec.scope; + let source = docs::for_source(&first.entry.spec.source, &first.entry.spec.scope); let mut text = format!( "---\n{FRONTMATTER_HEADER}title: C++ module ptfkit.{slug}\nnav-title: ptfkit.{slug}\n---\n\n" ); @@ -77,23 +91,23 @@ fn module(slug: &str, functions: &[&CompiledFunction]) -> Result { "# `ptfkit.{slug}`\n\n```cpp\nimport ptfkit.{slug};\n```\n\n" )); text.push_str(&format!("**Exported namespace:** `ptfkit::{slug}`\n\n")); - text.push_str(&format!("{}\n\n", escape_text(&source.summary))); + text.push_str(&format!("{}\n\n", escape_text(source.summary))); text.push_str("## Source\n\n"); - text.push_str(&escape_text(&source.citation_apa)); + text.push_str(&escape_text(source.reference.citation)); text.push_str("\n\n"); - if let Some(doi) = &source.doi { + if let Some(doi) = source.reference.doi { text.push_str(&format!( "[DOI: {}]({})\n\n", - escape_text(&doi.identifier), + escape_text(doi.identifier), doi.url )); } - if scope.territory.is_some() || scope.dataset.is_some() { + if source.territory.is_some() || source.dataset.is_some() { text.push_str("## Scope\n\n"); - if let Some(territory) = &scope.territory { + if let Some(territory) = source.territory { text.push_str(&format!("**Territory:** {}\n\n", escape_text(territory))); } - if let Some(dataset) = &scope.dataset { + if let Some(dataset) = source.dataset { text.push_str(&format!("**Dataset:** {}\n\n", escape_text(dataset))); } } @@ -137,14 +151,15 @@ fn structure(name: &str, fields: &[Parameter]) -> String { fn function_documentation(function: &CompiledFunction) -> Result { let spec = spec(function); + let document = docs::for_function(spec); let anchor = function_anchor(&function.core.name); let mut text = format!("### `{}` {{#{anchor}}}\n\n", function.core.name); - text.push_str(&format!("{}\n\n", escape_text(&spec.public_api.summary))); + text.push_str(&format!("{}\n\n", escape_text(document.summary))); text.push_str("```cpp\n"); text.push_str(&signature(function)?); text.push_str("\n```\n\n"); text.push_str("#### Parameters\n\n| Name | Description |\n| --- | --- |\n"); - for parameter in &spec.inputs { + for parameter in document.parameters { text.push_str(&format!( "| `{}` | {} |\n", parameter.name, @@ -152,17 +167,16 @@ fn function_documentation(function: &CompiledFunction) -> Result { )); } text.push_str("\n#### Returns\n\n"); - match &function.core.output { - Output::Scalar => text.push_str(&format!( - "{}\n\n", - parameter_details(&spec.outputs.fields()[0]) - )), - Output::Struct(_) => text.push_str(&format!("A `{}` value.\n\n", result_class(function)?)), + match document.returns { + docs::Returns::Scalar(field) => text.push_str(&format!("{}\n\n", parameter_details(field))), + docs::Returns::Record { .. } => { + text.push_str(&format!("A `{}` value.\n\n", result_class(function)?)) + } } - for note in &spec.documentation.notes { + for note in document.notes { admonition(&mut text, "note", note); } - for warning in &spec.documentation.warnings { + for warning in document.warnings { admonition(&mut text, "warning", warning); } Ok(text) @@ -176,7 +190,7 @@ fn functions_index(functions: &[(&str, &CompiledFunction)]) -> String { text.push_str(&format!( "| [`{qualified}`](modules/{slug}.md#{}) | {} | [`ptfkit.{slug}`](modules/{slug}.md) |\n", function_anchor(&function.core.name), - escape_table(&spec(function).public_api.summary), + escape_table(docs::for_function(spec(function)).summary), )); } text diff --git a/codegen/src/targets/documentation.rs b/codegen/src/targets/documentation.rs index 8e98234..2318648 100644 --- a/codegen/src/targets/documentation.rs +++ b/codegen/src/targets/documentation.rs @@ -1,6 +1,9 @@ use std::path::PathBuf; -use crate::model::{Entry, Function, Parameter}; +use crate::{ + documentation::{self as docs, FunctionDocument}, + model::{Entry, Parameter}, +}; pub(super) const HEADER: &str = "\n\n"; pub(super) const FRONTMATTER_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n\n"; @@ -36,14 +39,14 @@ fn render_index(entries: &[Entry]) -> String { ); text.push_str("| Source | Territory | Functions |\n| --- | --- | ---: |\n"); for entry in entries { - let spec = &entry.spec; - let territory = spec.scope.territory.as_deref().unwrap_or("—"); + let source = docs::for_source(&entry.spec.source, &entry.spec.scope); + let territory = source.territory.unwrap_or("—"); text.push_str(&format!( "| [{}](./{}.md) | {} | {} |\n", - escape_table(&spec.source.summary), + escape_table(source.summary), entry.slug, escape_table(territory), - spec.functions.len(), + entry.spec.functions.len(), )); } text @@ -51,56 +54,55 @@ fn render_index(entries: &[Entry]) -> String { fn render_source(entry: &Entry) -> String { let spec = &entry.spec; + let source = docs::for_source(&spec.source, &spec.scope); let mut text = format!( "---\n{FRONTMATTER_HEADER}title: PTF source {slug}\nnav-title: {slug}\n---\n\n", slug = entry.slug ); - text.push_str(&format!("# {}\n\n", spec.source.summary)); + text.push_str(&format!("# {}\n\n", source.summary)); text.push_str("## Source\n\n"); - text.push_str(&spec.source.citation_apa); + text.push_str(source.reference.citation); text.push_str("\n\n"); - if let Some(doi) = &spec.source.doi { + if let Some(doi) = source.reference.doi { text.push_str(&format!("[DOI: {}]({})\n\n", doi.identifier, doi.url)); } - if spec.scope.territory.is_some() || spec.scope.dataset.is_some() { + if source.territory.is_some() || source.dataset.is_some() { text.push_str("## Scope\n\n"); - if let Some(territory) = &spec.scope.territory { + if let Some(territory) = source.territory { text.push_str(&format!("**Territory:** {territory}\n\n")); } - if let Some(dataset) = &spec.scope.dataset { + if let Some(dataset) = source.dataset { text.push_str(&format!("**Dataset:** {dataset}\n\n")); } } text.push_str("## Functions\n\n"); for function in &spec.functions { - render_function(&mut text, function); + render_function( + &mut text, + docs::for_function(function), + &function.public_api.name, + &function.status, + ); } text } -fn render_function(text: &mut String, function: &Function) { - text.push_str(&format!("### `{}`\n\n", function.public_api.name)); - text.push_str(&function.public_api.summary); +fn render_function(text: &mut String, document: FunctionDocument<'_>, name: &str, status: &str) { + text.push_str(&format!("### `{name}`\n\n")); + text.push_str(document.summary); text.push_str("\n\n"); - text.push_str(&format!("**Status:** `{}`\n\n", function.status)); + text.push_str(&format!("**Status:** `{status}`\n\n")); text.push_str(&format!( "**Prediction target:** {}\n\n", - function.scope.prediction_target + document.remarks.prediction_target )); let models = [ - function - .scope + document .models .h_theta - .as_ref() .map(|model| format!("$h(\\theta)$ — {model}")), - function - .scope - .models - .k_h - .as_ref() - .map(|model| format!("$k(h)$ — {model}")), + document.models.k_h.map(|model| format!("$k(h)$ — {model}")), ] .into_iter() .flatten() @@ -109,12 +111,19 @@ fn render_function(text: &mut String, function: &Function) { text.push_str(&format!("**Models:** {}\n\n", models.join("; "))); } - render_parameters(text, "Inputs", &function.inputs); - render_parameters(text, "Outputs", function.outputs.fields()); - for note in &function.documentation.notes { + render_parameters(text, "Inputs", document.parameters); + render_parameters( + text, + "Outputs", + match document.returns { + docs::Returns::Scalar(field) => std::slice::from_ref(field), + docs::Returns::Record { fields, .. } => fields, + }, + ); + for note in document.notes { render_admonition(text, "note", note); } - for warning in &function.documentation.warnings { + for warning in document.warnings { render_admonition(text, "warning", warning); } } diff --git a/codegen/src/targets/native.rs b/codegen/src/targets/native.rs index 586f19e..54ff69f 100644 --- a/codegen/src/targets/native.rs +++ b/codegen/src/targets/native.rs @@ -4,6 +4,7 @@ use anyhow::Result; use convert_case::{Boundary, Case, Casing}; use crate::{ + documentation::{self as docs, FunctionDocument, SourceDocument}, model::{CompiledFunction, Function, Output, Parameter, Scope, Source}, render::{Render, Writer}, }; @@ -371,35 +372,43 @@ fn render_struct( } fn source_comment(source: &Source, scope: &Scope) -> Comment { + source_comment_from_document(docs::for_source(source, scope)) +} + +fn source_comment_from_document(document: SourceDocument<'_>) -> Comment { let mut lines = vec![ - format!("@brief {}", source.summary), + format!("@brief {}", document.summary), String::new(), "@details Source publication:".to_owned(), - source.citation_apa.clone(), + document.reference.citation.into(), ]; - if let Some(doi) = &source.doi { + if let Some(doi) = document.reference.doi { lines.push(format!("@see {} DOI: {}", doi.url, doi.identifier)); } - if let Some(territory) = &scope.territory { + if let Some(territory) = document.territory { lines.extend([ String::new(), "@remark Geographic scope:".to_owned(), - territory.clone(), + territory.into(), ]); } - if let Some(dataset) = &scope.dataset { + if let Some(dataset) = document.dataset { lines.extend([ String::new(), "@remark Calibration dataset:".to_owned(), - dataset.clone(), + dataset.into(), ]); } Comment(lines) } fn function_comment(function: &Function) -> Comment { - let mut lines = vec![format!("@brief {}", function.public_api.summary)]; - lines.extend(function.inputs.iter().map(|parameter| { + function_comment_from_document(docs::for_function(function)) +} + +fn function_comment_from_document(document: FunctionDocument<'_>) -> Comment { + let mut lines = vec![format!("@brief {}", document.summary)]; + lines.extend(document.parameters.iter().map(|parameter| { format!( "@param {} {}", parameter.name, @@ -407,7 +416,10 @@ fn function_comment(function: &Function) -> Comment { ) })); - let outputs = function.outputs.fields(); + let outputs = match document.returns { + docs::Returns::Scalar(field) => std::slice::from_ref(field), + docs::Returns::Record { fields, .. } => fields, + }; if outputs.len() == 1 { lines.push(format!( "@return {}", @@ -423,28 +435,21 @@ fn function_comment(function: &Function) -> Comment { ) })); } - if let Some(territory) = &function.scope.territory { + if let Some(territory) = document.territory { lines.extend([ String::new(), "@remark Geographic scope:".to_owned(), - territory.clone(), + territory.into(), ]); } lines.extend([ String::new(), "@details Prediction target:".to_owned(), - function.scope.prediction_target.clone(), + document.remarks.prediction_target.into(), ]); + lines.extend(document.notes.iter().map(|note| format!("@note {note}"))); lines.extend( - function - .documentation - .notes - .iter() - .map(|note| format!("@note {note}")), - ); - lines.extend( - function - .documentation + document .warnings .iter() .map(|warning| format!("@warning {warning}")), diff --git a/codegen/src/targets/py/wrapper.rs b/codegen/src/targets/py/wrapper.rs index 3a326cb..16944eb 100644 --- a/codegen/src/targets/py/wrapper.rs +++ b/codegen/src/targets/py/wrapper.rs @@ -2,7 +2,10 @@ use std::collections::BTreeMap; use anyhow::{Result, bail}; -use crate::model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}; +use crate::{ + documentation::{self as docs, FunctionDocument}, + model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}, +}; use super::{super::documentation, WRAPPER_HEADER, natural_sort_key}; @@ -221,22 +224,23 @@ fn view(resolved: &CompiledFunction) -> PythonFunction<'_> { } fn module_docstring(source: &Source, scope: &Scope) -> String { + let document = docs::for_source(source, scope); let mut lines = vec![ - format!("r\"\"\"{}", source.summary), + format!("r\"\"\"{}", document.summary), String::new(), "Reference:".into(), ]; - lines.extend(wrap_markdown_block(&source.citation_apa, " ")); - if let Some(doi) = &source.doi { + lines.extend(wrap_markdown_block(document.reference.citation, " ")); + if let Some(doi) = document.reference.doi { lines.extend(wrap_markdown_block( &format!("[DOI: {}]({})", doi.identifier, doi.url), " ", )); } - if let Some(territory) = &scope.territory { + if let Some(territory) = document.territory { definition_list_block(&mut lines, "Territory", territory); } - if let Some(dataset) = &scope.dataset { + if let Some(dataset) = document.dataset { definition_list_block(&mut lines, "Dataset", dataset); } lines.push(String::new()); @@ -245,41 +249,42 @@ fn module_docstring(source: &Source, scope: &Scope) -> String { } fn function_docstring(function: &Function) -> String { - let mut arguments = function - .inputs + let document = docs::for_function(function); + function_docstring_from_document(document, function.result_class()) +} + +fn function_docstring_from_document( + document: FunctionDocument<'_>, + result_class: Option<&str>, +) -> String { + let mut arguments = document + .parameters .iter() .map(parameter_documentation) .collect::>(); arguments.push("out: Optional output arrays for in-place calculation.".into()); - let returns = if let Some(result_class) = function.result_class() { + let returns = if let Some(result_class) = result_class { vec![format!( "{result_class}: Results grouped by result attributes." )] } else { - function - .outputs - .fields() - .iter() - .map(parameter_documentation) - .collect() + match document.returns { + docs::Returns::Scalar(field) => vec![parameter_documentation(field)], + docs::Returns::Record { fields, .. } => { + fields.iter().map(parameter_documentation).collect() + } + } }; let mut sections = vec![("Arguments", arguments), ("Returns", returns)]; - if let Some(territory) = &function.scope.territory { - sections.push(("Territory", vec![territory.clone()])); + if let Some(territory) = document.territory { + sections.push(("Territory", vec![territory.into()])); } let models = [ - function - .scope + document .models .h_theta - .as_ref() .map(|model| format!("$h(\\theta)$: {model}")), - function - .scope - .models - .k_h - .as_ref() - .map(|model| format!("$k(h)$: {model}")), + document.models.k_h.map(|model| format!("$k(h)$: {model}")), ] .into_iter() .flatten() @@ -291,15 +296,15 @@ fn function_docstring(function: &Function) -> String { "Notes", std::iter::once(format!( "Prediction target: {}", - function.scope.prediction_target + document.remarks.prediction_target )) - .chain(function.documentation.notes.iter().cloned()) + .chain(document.notes.iter().cloned()) .collect(), )); - if !function.documentation.warnings.is_empty() { - sections.push(("Warning", function.documentation.warnings.clone())); + if !document.warnings.is_empty() { + sections.push(("Warning", document.warnings.to_vec())); } - render_docstring(" ", &function.public_api.summary, §ions, 4) + render_docstring(" ", document.summary, §ions, 4) } fn result_class_docstring(function: &Function) -> String { From 768eb364a95045a7613e73ad5ff469263d71618f Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 23:02:05 +0300 Subject: [PATCH 07/14] refactor(codegen): structure Python generation --- codegen/src/targets/c_expression.rs | 21 ++-- codegen/src/targets/py/c.rs | 145 +++++++++++++++++----------- codegen/src/targets/py/mod.rs | 1 + codegen/src/targets/py/stub.rs | 20 ++-- codegen/src/targets/py/syntax.rs | 61 ++++++++++++ codegen/src/targets/py/test.rs | 131 ++++++++++++------------- codegen/src/targets/py/wrapper.rs | 105 ++++++++++++-------- 7 files changed, 300 insertions(+), 184 deletions(-) create mode 100644 codegen/src/targets/py/syntax.rs diff --git a/codegen/src/targets/c_expression.rs b/codegen/src/targets/c_expression.rs index f64a1b5..32200b7 100644 --- a/codegen/src/targets/c_expression.rs +++ b/codegen/src/targets/c_expression.rs @@ -8,15 +8,6 @@ pub(super) enum Dialect { Cpp, } -pub(super) fn render( - value: &Expr, - inputs: &[String], - variables: &[Variable], - dialect: Dialect, -) -> String { - expression(value, inputs, variables, dialect).to_string() -} - pub(super) fn expression<'a>( value: &'a Expr, inputs: &'a [String], @@ -243,11 +234,11 @@ mod tests { }; assert_eq!( - render(&expression, &inputs(), &[], Dialect::C), + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), "x - (y - z)" ); assert_eq!( - render(&expression, &inputs(), &[], Dialect::Cpp), + super::expression(&expression, &inputs(), &[], Dialect::Cpp).to_string(), "x - (y - z)" ); } @@ -276,7 +267,7 @@ mod tests { }; assert_eq!( - render(&expression, &inputs(), &[], Dialect::C), + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), "-(x + y) / (z * (x - y))" ); } @@ -297,7 +288,7 @@ mod tests { }; assert_eq!( - render(&expression, &inputs(), &[], Dialect::C), + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), "(x + y) * z" ); } @@ -331,11 +322,11 @@ mod tests { }; assert_eq!( - render(&expression, &inputs(), &[], Dialect::C), + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), "fmin(pow(x + y, -z), sqrt(x * y))" ); assert_eq!( - render(&expression, &inputs(), &[], Dialect::Cpp), + super::expression(&expression, &inputs(), &[], Dialect::Cpp).to_string(), "std::fmin(std::pow(x + y, -z), std::sqrt(x * y))" ); } diff --git a/codegen/src/targets/py/c.rs b/codegen/src/targets/py/c.rs index a58a0c7..35e64cb 100644 --- a/codegen/src/targets/py/c.rs +++ b/codegen/src/targets/py/c.rs @@ -1,6 +1,7 @@ use anyhow::Result; use crate::model::{CompiledFunction, Output}; +use crate::render::Writer; use super::{ super::{ @@ -11,67 +12,72 @@ use super::{ }; pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { - let mut includes = String::new(); + let mut includes = Writer::new(); let mut registers = Vec::new(); let mut writes = Vec::new(); for (slug, functions) in group_by_source(functions) { let register = format!("ptfkit_register_{slug}"); registers.push(register.clone()); - let mut definitions = String::new(); - let mut calls = String::new(); + let mut definitions = Writer::new(); + let mut calls = Writer::new(); for function in functions { - definitions.push_str(&ufunc(function)?); - calls.push_str(&format!( - " if (ptfkit_add_ufunc(module, \"{name}\", {name}_functions, {name}_types, {nin}, {nout}) < 0) return -1;\n", + definitions.write(ufunc(function)?); + calls.line(format_args!( + "if (ptfkit_add_ufunc(module, \"{name}\", {name}_functions, {name}_types, {nin}, {nout}) < 0) return -1;", name = function.core.name, nin = function.core.inputs.len(), nout = output_count(&function.core.output), )); } - includes.push_str(&format!("#include \"{slug}.c\"\n")); - writes.push(( - format!("src/ptfkit/{slug}.c"), - format!( - "{C_HEADER}#include \"ufunc.h\"\n\n{definitions}int {register}(PyObject *module) {{\n{calls} return 0;\n}}\n" - ), + includes.line(format_args!("#include \"{slug}.c\"")); + let mut source = Writer::new(); + source.write(C_HEADER); + source.line("#include \"ufunc.h\""); + source.blank_line(); + source.write(definitions.into_string()); + source.line(format_args!("int {register}(PyObject *module) {{")); + source.indented(|writer| { + writer.write(calls.into_string()); + writer.line("return 0;"); + }); + source.line("}"); + writes.push((format!("src/ptfkit/{slug}.c"), source.into_string())); + } + let mut calls = Writer::new(); + for register in ®isters { + calls.line(format_args!( + "if ({register}(module) < 0) {{ Py_DECREF(module); return NULL; }}" )); } - let calls = registers - .iter() - .map(|register| { - format!(" if ({register}(module) < 0) {{ Py_DECREF(module); return NULL; }}\n") - }) - .collect::(); - writes.push(( - "src/ptfkit/ptfkit.c".into(), - format!( - "{C_HEADER}#define PY_SSIZE_T_CLEAN\n#define PY_ARRAY_UNIQUE_SYMBOL PTFKIT_ARRAY_API\n#include \n#include \n#include \n\n{includes}\nstatic struct PyModuleDef module_def = {{ PyModuleDef_HEAD_INIT, \"_ptfkit\", NULL, -1, NULL }};\n\nPyMODINIT_FUNC PyInit__ptfkit(void) {{\n PyObject *module = PyModule_Create(&module_def);\n if (module == NULL) return NULL;\n import_array();\n import_ufunc();\n{calls} return module;\n}}\n" - ), - )); + let mut entry = Writer::new(); + entry.write(C_HEADER); + entry.line("#define PY_SSIZE_T_CLEAN"); + entry.line("#define PY_ARRAY_UNIQUE_SYMBOL PTFKIT_ARRAY_API"); + entry.line("#include "); + entry.line("#include "); + entry.line("#include "); + entry.blank_line(); + entry.write(includes.into_string()); + entry.blank_line(); + entry.line("static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, \"_ptfkit\", NULL, -1, NULL };"); + entry.blank_line(); + entry.line("PyMODINIT_FUNC PyInit__ptfkit(void) {"); + entry.indented(|writer| { + writer.line("PyObject *module = PyModule_Create(&module_def);"); + writer.line("if (module == NULL) return NULL;"); + writer.line("import_array();"); + writer.line("import_ufunc();"); + writer.write(calls.into_string()); + writer.line("return module;"); + }); + entry.line("}"); + writes.push(("src/ptfkit/ptfkit.c".into(), entry.into_string())); Ok(writes) } fn ufunc(function: &CompiledFunction) -> Result { let name = &function.core.name; let inputs = &function.core.inputs; - let mut locals = String::new(); - for (index, input) in inputs.iter().enumerate() { - locals.push_str(&format!( - " const double {input} = *(const double *)args[{index}];\n" - )); - } - for variable in &function.ir.variables { - locals.push_str(&format!( - " const double {} = {};\n", - variable.name, - c_expression::render( - &variable.expression, - inputs, - &function.ir.variables, - Dialect::C, - ) - )); - } let values = match &function.core.output { Output::Scalar => vec![ function.entry.spec.functions[function.function_index] @@ -82,23 +88,50 @@ fn ufunc(function: &CompiledFunction) -> Result { ], Output::Struct(fields) => fields.clone(), }; - let writes = values - .iter() - .enumerate() - .map(|(index, value)| { - format!( - " *(double *)args[{}] = {value};\n", - inputs.len() + index - ) - }) - .collect::(); let types = std::iter::repeat_n("NPY_DOUBLE", inputs.len() + values.len()) .collect::>() .join(", "); - Ok(format!( - "static void {name}_loop(char **args, const npy_intp *dimensions, const npy_intp *steps, void *data) {{\n npy_intp index;\n for (index = 0; index < dimensions[0]; index++) {{\n{locals}{writes} for (int arg = 0; arg < {}; arg++) args[arg] += steps[arg];\n }}\n}}\nstatic PyUFuncGenericFunction {name}_functions[] = {{ {name}_loop }};\nstatic char {name}_types[] = {{ {types} }};\n\n", - inputs.len() + values.len() - )) + let mut writer = Writer::new(); + writer.line(format_args!("static void {name}_loop(char **args, const npy_intp *dimensions, const npy_intp *steps, void *data) {{")); + writer.indented(|writer| { + writer.line("npy_intp index;"); + writer.line("for (index = 0; index < dimensions[0]; index++) {"); + writer.indented(|writer| { + for (index, input) in inputs.iter().enumerate() { + writer.line(format_args!( + "const double {input} = *(const double *)args[{index}];" + )); + } + for variable in &function.ir.variables { + writer.write(format_args!("const double {} = ", variable.name)); + writer.write(c_expression::expression( + &variable.expression, + inputs, + &function.ir.variables, + Dialect::C, + )); + writer.line(";"); + } + for (index, value) in values.iter().enumerate() { + writer.line(format_args!( + "*(double *)args[{}] = {value};", + inputs.len() + index + )); + } + writer.line(format_args!( + "for (int arg = 0; arg < {}; arg++) args[arg] += steps[arg];", + inputs.len() + values.len() + )); + }); + writer.line("}"); + }); + writer.line("}"); + writer.line(format_args!( + "static PyUFuncGenericFunction {name}_functions[] = {{ {name}_loop }};" + )); + writer.line(format_args!("static char {name}_types[] = {{ {types} }};")); + writer.blank_line(); + Ok(writer.into_string()) } fn output_count(output: &Output) -> usize { diff --git a/codegen/src/targets/py/mod.rs b/codegen/src/targets/py/mod.rs index fc6d99d..bb003b8 100644 --- a/codegen/src/targets/py/mod.rs +++ b/codegen/src/targets/py/mod.rs @@ -1,5 +1,6 @@ mod c; mod stub; +mod syntax; mod test; mod wrapper; diff --git a/codegen/src/targets/py/stub.rs b/codegen/src/targets/py/stub.rs index 36fcad1..ee37283 100644 --- a/codegen/src/targets/py/stub.rs +++ b/codegen/src/targets/py/stub.rs @@ -1,4 +1,6 @@ -use crate::{model::CompiledFunction, targets::py::WRAPPER_HEADER}; +use crate::model::CompiledFunction; + +use super::{WRAPPER_HEADER, syntax::Module}; pub(super) fn render(functions: &[CompiledFunction]) -> String { let mut names = functions @@ -7,14 +9,14 @@ pub(super) fn render(functions: &[CompiledFunction]) -> String { .collect::>(); names.sort_unstable(); - let mut lines = vec![ - WRAPPER_HEADER.trim_end().to_owned(), - String::new(), - "from numpy import ufunc".into(), - String::new(), - ]; - lines.extend(names.into_iter().map(|name| format!("{name}: ufunc"))); - format!("{}\n", lines.join("\n")) + let mut module = Module::new(WRAPPER_HEADER); + module.blank_line(); + module.import("numpy", "ufunc"); + module.blank_line(); + for name in names { + module.line(format_args!("{name}: ufunc")); + } + module.into_string() } #[cfg(test)] diff --git a/codegen/src/targets/py/syntax.rs b/codegen/src/targets/py/syntax.rs new file mode 100644 index 0000000..8b59c42 --- /dev/null +++ b/codegen/src/targets/py/syntax.rs @@ -0,0 +1,61 @@ +use std::fmt::Display; + +use crate::render::Writer; + +/// Small Python-specific rendering helpers for the declarations emitted by the +/// public wrapper and its generated tests. This deliberately models only the +/// constructs the generator owns, not Python's complete syntax. +pub(super) struct Module { + writer: Writer, +} + +impl Module { + pub(super) fn new(header: &str) -> Self { + let mut writer = Writer::new(); + writer.write(header.trim_end()); + Self { writer } + } + + pub(super) fn blank_line(&mut self) { + self.writer.blank_line(); + } + + pub(super) fn line(&mut self, value: impl Display) { + self.writer.line(value); + } + + pub(super) fn write(&mut self, value: impl Display) { + self.writer.write(value); + } + + pub(super) fn future_annotations(&mut self) { + self.line("from __future__ import annotations"); + } + + pub(super) fn import(&mut self, module: &str, names: impl Display) { + self.line(format_args!("from {module} import {names}")); + } + + pub(super) fn assignment(&mut self, name: &str, value: impl Display) { + self.line(format_args!("{name} = {value}")); + } + + pub(super) fn block( + &mut self, + opening: impl Display, + render: impl FnOnce(&mut Writer), + closing: impl Display, + ) { + self.writer.line(opening); + self.writer.indented(render); + self.writer.line(closing); + } + + pub(super) fn indented(&mut self, render: impl FnOnce(&mut Writer)) { + self.writer.indented(render); + } + + pub(super) fn into_string(self) -> String { + self.writer.into_string() + } +} diff --git a/codegen/src/targets/py/test.rs b/codegen/src/targets/py/test.rs index 0e2d9dc..42422b7 100644 --- a/codegen/src/targets/py/test.rs +++ b/codegen/src/targets/py/test.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use crate::model::{CompiledFunction, Function, Outputs, PythonGeneration}; -use super::{WRAPPER_HEADER, natural_sort_key}; +use super::{WRAPPER_HEADER, natural_sort_key, syntax::Module}; pub(super) fn render(functions: &[CompiledFunction]) -> Vec<(String, PythonGeneration, String)> { let mut modules: BTreeMap> = BTreeMap::new(); @@ -38,54 +38,58 @@ fn module_source(slug: &str, functions: &[&CompiledFunction]) -> String { } imports.sort_by_key(|name| natural_sort_key(name)); imports.dedup(); - let imports = imports.join(", "); - let tests = functions - .iter() - .map(|resolved| { - let function = &resolved.entry.spec.functions[resolved.function_index]; - function_source(function) - }) - .collect::>() - .join("\n\n\n"); - - format!( - "{header}\nfrom __future__ import annotations\n\nimport pytest\n\nfrom _helpers import prepare_vector_case\nfrom ptfkit.{slug} import {imports}\n\n\n{tests}\n", - header = WRAPPER_HEADER.trim_end(), - ) + let mut module = Module::new(WRAPPER_HEADER); + module.line(""); + module.future_annotations(); + module.blank_line(); + module.line("import pytest"); + module.blank_line(); + module.import("_helpers", "prepare_vector_case"); + module.import(&format!("ptfkit.{slug}"), imports.join(", ")); + module.blank_line(); + module.blank_line(); + for (index, resolved) in functions.iter().enumerate() { + if index > 0 { + module.blank_line(); + module.blank_line(); + } + let function = &resolved.entry.spec.functions[resolved.function_index]; + function_source(&mut module, function); + } + module.into_string() } -fn function_source(function: &Function) -> String { +fn function_source(module: &mut Module, function: &Function) { let cases_name = format!("CASES_{}", function.public_api.name.to_ascii_uppercase()); - let cases = function - .golden_tests - .iter() - .map(|case| { - format!( - " ({inputs}, {expected}, {rtol}, {atol}),", - inputs = dictionary(&case.inputs), - expected = dictionary(&case.expected), - rtol = float(case.rtol), - atol = float(case.atol), - ) - }) - .collect::>() - .join("\n"); - let assertion = expected_assertion(function, " ", ""); - let vector_tests = if !function.golden_tests.is_empty() { - vector_test_source(function, &cases_name) - } else { - Default::default() - }; - format!( - "{cases_name} = [\n{cases}\n]\n\n\n@pytest.mark.parametrize(('inputs', 'expected', 'rtol', 'atol'), {cases_name})\ndef test_{name}_golden(inputs: dict[str, float], expected: dict[str, float], rtol: float, atol: float):\n result = {name}(**inputs)\n\n{assertion}{vector_tests}", - cases_name = cases_name, - name = function.public_api.name, - assertion = assertion, - vector_tests = vector_tests, - ) + module.assignment(&cases_name, "["); + for case in &function.golden_tests { + module.line(format_args!( + " ({}, {}, {}, {}),", + dictionary(&case.inputs), + dictionary(&case.expected), + float(case.rtol), + float(case.atol), + )); + } + module.line("]"); + module.blank_line(); + module.blank_line(); + let name = &function.public_api.name; + module.line(format_args!( + "@pytest.mark.parametrize(('inputs', 'expected', 'rtol', 'atol'), {cases_name})" + )); + module.line(format_args!( + "def test_{name}_golden(inputs: dict[str, float], expected: dict[str, float], rtol: float, atol: float):" + )); + module.line(format_args!(" result = {name}(**inputs)")); + module.line(""); + module.line(expected_assertion(function, " ", "")); + if !function.golden_tests.is_empty() { + vector_test_source(module, function, &cases_name); + } } -fn vector_test_source(function: &Function, cases_name: &str) -> String { +fn vector_test_source(module: &mut Module, function: &Function, cases_name: &str) { let array_assertion = expected_assertion(function, " ", "[0]"); let out_assertion = match &function.outputs { Outputs::Scalar { .. } => " assert result is out", @@ -97,27 +101,24 @@ fn vector_test_source(function: &Function, cases_name: &str) -> String { .result_class() .map(|result_class| format!(", {result_class}")) .unwrap_or_default(); - format!( - r#" - - -def test_{name}_array(): - inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls}) - result = {name}(**inputs, out=None) -{array_assertion} - - -def test_{name}_out(): - inputs, expected, rtol, atol, out = prepare_vector_case({cases_name}{result_cls}) - result = {name}(**inputs, out=out) -{out_assertion} -{array_assertion}"#, - name = function.public_api.name, - cases_name = cases_name, - result_cls = result_cls, - array_assertion = array_assertion, - out_assertion = out_assertion, - ) + let name = &function.public_api.name; + module.blank_line(); + module.blank_line(); + module.line(format_args!("def test_{name}_array():")); + module.line(format_args!( + " inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls})" + )); + module.line(format_args!(" result = {name}(**inputs, out=None)")); + module.line(&array_assertion); + module.blank_line(); + module.blank_line(); + module.line(format_args!("def test_{name}_out():")); + module.line(format_args!( + " inputs, expected, rtol, atol, out = prepare_vector_case({cases_name}{result_cls})" + )); + module.line(format_args!(" result = {name}(**inputs, out=out)")); + module.line(out_assertion); + module.line(array_assertion); } fn expected_assertion(function: &Function, indent: &str, index: &str) -> String { diff --git a/codegen/src/targets/py/wrapper.rs b/codegen/src/targets/py/wrapper.rs index 16944eb..b1747df 100644 --- a/codegen/src/targets/py/wrapper.rs +++ b/codegen/src/targets/py/wrapper.rs @@ -7,7 +7,7 @@ use crate::{ model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}, }; -use super::{super::documentation, WRAPPER_HEADER, natural_sort_key}; +use super::{super::documentation, WRAPPER_HEADER, natural_sort_key, syntax::Module}; struct PythonFunction<'a> { name: &'a str, @@ -107,7 +107,7 @@ fn module_source( functions: &[PythonFunction<'_>], exports: &[String], ) -> String { - let mut sections = vec![WRAPPER_HEADER.trim_end().into()]; + let mut module = Module::new(WRAPPER_HEADER); if functions.len() > 1 { let has_long_import = functions.iter().any(|function| { format!( @@ -118,48 +118,75 @@ fn module_source( .count() > 100 }); - sections.push(if has_long_import { - "# ruff: noqa: E501, I001".into() + module.blank_line(); + module.line(if has_long_import { + "# ruff: noqa: E501, I001" } else { - "# ruff: noqa: I001".into() + "# ruff: noqa: I001" }); } - sections.extend([ - module_docstring(source, scope), - "from __future__ import annotations".into(), - format!("from typing import {typing_imports}"), - "from ptfkit._ptfkit import (".into(), - ]); - sections.extend(functions.iter().map(|function| { - format!( - " {rust_name} as _{rust_name},", - rust_name = function.rust_name - ) - })); - sections.extend([ - ")\n".into(), - "if TYPE_CHECKING:\n from numpy import floating\n from numpy.typing import ArrayLike, NDArray" - .into(), - ]); + module.blank_line(); + module.write(module_docstring(source, scope)); + module.blank_line(); + module.future_annotations(); + module.blank_line(); + module.import("typing", typing_imports); + module.blank_line(); + module.block( + "from ptfkit._ptfkit import (", + |writer| { + for function in functions { + writer.line(format_args!( + "{rust_name} as _{rust_name},", + rust_name = function.rust_name + )); + } + }, + ")", + ); + module.blank_line(); + module.blank_line(); + module.line("if TYPE_CHECKING:"); + module.indented(|writer| { + writer.line("from numpy import floating"); + writer.line("from numpy.typing import ArrayLike, NDArray"); + }); if !classes.is_empty() { - sections.push("T = TypeVar('T')".into()); - sections.extend(classes.iter().map(|class| { - format!( - "class {}(NamedTuple, Generic[T]):\n{}\n{}", - class.name, class.docstring, class.field_definitions - ) - })); + module.blank_line(); + module.assignment("T", "TypeVar('T')"); + for class in classes { + module.blank_line(); + module.line(format_args!( + "class {}(NamedTuple, Generic[T]):", + class.name + )); + module.indented(|writer| { + writer.write(&class.docstring); + writer.write("\n"); + writer.write(&class.field_definitions); + writer.write("\n"); + }); + } } - sections.push(format!( - "__all__ = [{}]", - exports - .iter() - .map(|export| format!("'{export}'")) - .collect::>() - .join(", ") - )); - sections.extend(functions.iter().map(function_source)); - format!("{}\n", sections.join("\n\n")) + module.blank_line(); + module.assignment( + "__all__", + format_args!( + "[{}]", + exports + .iter() + .map(|export| format!("'{export}'")) + .collect::>() + .join(", ") + ), + ); + for function in functions { + module.blank_line(); + module.blank_line(); + module.write(function_source(function)); + } + module.line(""); + module.into_string() } fn function_source(function: &PythonFunction<'_>) -> String { From 15927f5932811c257f1665d531996909336b6ced Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 23:17:17 +0300 Subject: [PATCH 08/14] refactor(codegen): declare output layouts --- codegen/src/targets/mod.rs | 309 ++++++++++++++++++++--------------- codegen/src/targets/write.rs | 50 +++--- 2 files changed, 206 insertions(+), 153 deletions(-) diff --git a/codegen/src/targets/mod.rs b/codegen/src/targets/mod.rs index 6a4a57d..ee59b6f 100644 --- a/codegen/src/targets/mod.rs +++ b/codegen/src/targets/mod.rs @@ -32,128 +32,154 @@ pub(super) fn group_by_source( } #[derive(Clone, Copy, PartialEq, Eq)] -pub(super) enum Target { - Documentation, - CDocumentation, - CppDocumentation, - PythonDocumentation, +pub(super) enum Formatter { + None, Rust, - PythonExtension, - PythonWrapper, - PythonTest, - NativeC, - NativeCppModule, - NativeCppCmake, - NativeCTest, - NativeCppTest, + Python, + C, + Cpp, } -impl Target { - pub(super) const ALL: [Self; 13] = [ - Self::Documentation, - Self::CDocumentation, - Self::CppDocumentation, - Self::PythonDocumentation, - Self::Rust, - Self::PythonExtension, - Self::PythonWrapper, - Self::PythonTest, - Self::NativeC, - Self::NativeCppModule, - Self::NativeCppCmake, - Self::NativeCTest, - Self::NativeCppTest, - ]; - - fn output_path(self, root: &Path, relative: &Path) -> PathBuf { - match self { - Self::Documentation => root.join("docs/src/ptf-catalog/sources").join(relative), - Self::CDocumentation => root.join("docs/src/reference/c").join(relative), - Self::CppDocumentation => root.join("docs/src/reference/cpp").join(relative), - Self::PythonDocumentation => root.join("docs/src/reference/python").join(relative), - Self::Rust => root.join("targets/ptfkit-rs/src").join(relative), - Self::PythonExtension => root.join("targets/ptfkit-py").join(relative), - Self::PythonWrapper => root.join("targets/ptfkit-py/src").join(relative), - Self::PythonTest => root.join("targets/ptfkit-py").join(relative), - Self::NativeC => root.join("targets/ptfkit-native/include").join(relative), - Self::NativeCppModule => root.join("targets/ptfkit-native/cpp").join(relative), - Self::NativeCppCmake => root.join("targets/ptfkit-native/cmake").join(relative), - Self::NativeCTest => root.join("targets/ptfkit-native/tests/c").join(relative), - Self::NativeCppTest => root.join("targets/ptfkit-native/tests/cpp").join(relative), - } - } - - fn cleanup_directory(self, root: &Path) -> PathBuf { - match self { - Self::Documentation => root.join("docs/src/ptf-catalog/sources"), - Self::CDocumentation => root.join("docs/src/reference/c"), - Self::CppDocumentation => root.join("docs/src/reference/cpp"), - Self::PythonDocumentation => root.join("docs/src/reference/python"), - Self::Rust => root.join("targets/ptfkit-rs/src"), - Self::PythonExtension | Self::PythonWrapper => { - root.join("targets/ptfkit-py/src/ptfkit") - } - Self::PythonTest => root.join("targets/ptfkit-py/tests"), - Self::NativeC => root.join("targets/ptfkit-native/include"), - Self::NativeCppModule => root.join("targets/ptfkit-native/cpp"), - Self::NativeCppCmake => root.join("targets/ptfkit-native/cmake"), - Self::NativeCTest => root.join("targets/ptfkit-native/tests/c"), - Self::NativeCppTest => root.join("targets/ptfkit-native/tests/cpp"), - } - } +pub(super) struct Layout { + pub(super) output_directory: &'static str, + pub(super) cleanup_directory: &'static str, + pub(super) generated_header: &'static str, + pub(super) formatter: Formatter, +} - fn generated_header(self) -> &'static str { - match self { - Self::Documentation => documentation::HEADER, - Self::CDocumentation => documentation::HEADER, - Self::CppDocumentation => documentation::HEADER, - Self::PythonDocumentation => documentation::HEADER, - Self::Rust => rs::HEADER, - Self::PythonExtension => py::C_HEADER, - Self::PythonWrapper => py::WRAPPER_HEADER, - Self::PythonTest => py::WRAPPER_HEADER, - Self::NativeC | Self::NativeCppModule | Self::NativeCTest | Self::NativeCppTest => { - native::HEADER - } - Self::NativeCppCmake => native::CMAKE_HEADER, - } - } +macro_rules! layout { + ( + $name:ident, + $output_directory:literal, + $cleanup_directory:literal, + $generated_header:path, + $formatter:path, + ) => { + pub(super) static $name: Layout = Layout { + output_directory: $output_directory, + cleanup_directory: $cleanup_directory, + generated_header: $generated_header, + formatter: $formatter, + }; + }; +} - fn is_clang_formatted(self) -> bool { - matches!( - self, - Self::PythonExtension - | Self::NativeC - | Self::NativeCppModule - | Self::NativeCTest - | Self::NativeCppTest - ) - } +layout!( + DOCUMENTATION, + "docs/src/ptf-catalog/sources", + "docs/src/ptf-catalog/sources", + documentation::HEADER, + Formatter::None, +); +layout!( + C_DOCUMENTATION, + "docs/src/reference/c", + "docs/src/reference/c", + documentation::HEADER, + Formatter::None, +); +layout!( + CPP_DOCUMENTATION, + "docs/src/reference/cpp", + "docs/src/reference/cpp", + documentation::HEADER, + Formatter::None, +); +layout!( + PYTHON_DOCUMENTATION, + "docs/src/reference/python", + "docs/src/reference/python", + documentation::HEADER, + Formatter::None, +); +layout!( + RUST, + "targets/ptfkit-rs/src", + "targets/ptfkit-rs/src", + rs::HEADER, + Formatter::Rust, +); +layout!( + PYTHON_EXTENSION, + "targets/ptfkit-py", + "targets/ptfkit-py/src/ptfkit", + py::C_HEADER, + Formatter::C, +); +layout!( + PYTHON_WRAPPER, + "targets/ptfkit-py/src", + "targets/ptfkit-py/src/ptfkit", + py::WRAPPER_HEADER, + Formatter::Python, +); +layout!( + PYTHON_TEST, + "targets/ptfkit-py", + "targets/ptfkit-py/tests", + py::WRAPPER_HEADER, + Formatter::Python, +); +layout!( + NATIVE_C, + "targets/ptfkit-native/include", + "targets/ptfkit-native/include", + native::HEADER, + Formatter::Cpp, +); +layout!( + NATIVE_CPP_MODULE, + "targets/ptfkit-native/cpp", + "targets/ptfkit-native/cpp", + native::HEADER, + Formatter::Cpp, +); +layout!( + NATIVE_CPP_CMAKE, + "targets/ptfkit-native/cmake", + "targets/ptfkit-native/cmake", + native::CMAKE_HEADER, + Formatter::None, +); +layout!( + NATIVE_C_TEST, + "targets/ptfkit-native/tests/c", + "targets/ptfkit-native/tests/c", + native::HEADER, + Formatter::C, +); +layout!( + NATIVE_CPP_TEST, + "targets/ptfkit-native/tests/cpp", + "targets/ptfkit-native/tests/cpp", + native::HEADER, + Formatter::Cpp, +); - fn format(self, root: &Path, paths: &[PathBuf]) -> Result<()> { - match self { - Self::Documentation - | Self::CDocumentation - | Self::CppDocumentation - | Self::PythonDocumentation => Ok(()), - Self::Rust => write::format_rust(paths), - Self::PythonExtension | Self::NativeCTest => write::format_c(paths), - Self::PythonWrapper => write::format_python(root, paths), - Self::PythonTest => write::format_python(root, paths), - Self::NativeC | Self::NativeCppModule | Self::NativeCppTest => write::format_cpp(paths), - Self::NativeCppCmake => Ok(()), - } - } -} +pub(super) const LAYOUTS: [&Layout; 13] = [ + &DOCUMENTATION, + &C_DOCUMENTATION, + &CPP_DOCUMENTATION, + &PYTHON_DOCUMENTATION, + &RUST, + &PYTHON_EXTENSION, + &PYTHON_WRAPPER, + &PYTHON_TEST, + &NATIVE_C, + &NATIVE_CPP_MODULE, + &NATIVE_CPP_CMAKE, + &NATIVE_C_TEST, + &NATIVE_CPP_TEST, +]; -pub(super) struct TargetOutput { - pub(super) target: Target, +pub(super) struct Output { + pub(super) layout: &'static Layout, pub(super) files: Vec, } -impl TargetOutput { - fn new(target: Target, files: Vec) -> Self { - Self { target, files } +impl Output { + fn new(layout: &'static Layout, files: Vec) -> Self { + Self { layout, files } } } @@ -209,19 +235,19 @@ pub(crate) fn run(root: &Path, entries: Vec) -> Result<()> { write::commit( root, &[ - TargetOutput::new(Target::Documentation, documentation), - TargetOutput::new(Target::CDocumentation, c_documentation), - TargetOutput::new(Target::CppDocumentation, cpp_documentation), - TargetOutput::new(Target::PythonDocumentation, python_documentation), - TargetOutput::new(Target::Rust, rust), - TargetOutput::new(Target::PythonExtension, c), - TargetOutput::new(Target::PythonWrapper, wrappers), - TargetOutput::new(Target::PythonTest, tests), - TargetOutput::new(Target::NativeC, native.c_headers), - TargetOutput::new(Target::NativeCppModule, native.cpp_modules), - TargetOutput::new(Target::NativeCppCmake, native.cpp_cmake), - TargetOutput::new(Target::NativeCTest, native.c_tests), - TargetOutput::new(Target::NativeCppTest, native.cpp_tests), + Output::new(&DOCUMENTATION, documentation), + Output::new(&C_DOCUMENTATION, c_documentation), + Output::new(&CPP_DOCUMENTATION, cpp_documentation), + Output::new(&PYTHON_DOCUMENTATION, python_documentation), + Output::new(&RUST, rust), + Output::new(&PYTHON_EXTENSION, c), + Output::new(&PYTHON_WRAPPER, wrappers), + Output::new(&PYTHON_TEST, tests), + Output::new(&NATIVE_C, native.c_headers), + Output::new(&NATIVE_CPP_MODULE, native.cpp_modules), + Output::new(&NATIVE_CPP_CMAKE, native.cpp_cmake), + Output::new(&NATIVE_C_TEST, native.c_tests), + Output::new(&NATIVE_CPP_TEST, native.cpp_tests), ], ) } @@ -237,24 +263,41 @@ pub(crate) fn check_generated(root: &Path, entries: Vec) -> Result<()> { mod tests { use std::path::Path; - use super::Target; + use super::{ + C_DOCUMENTATION, CPP_DOCUMENTATION, DOCUMENTATION, PYTHON_DOCUMENTATION, PYTHON_EXTENSION, + PYTHON_WRAPPER, + }; #[test] fn documentation_targets_use_structured_mkdocs_roots() { let root = Path::new("repository"); let relative = Path::new("index.md"); - for (target, directory) in [ - (Target::Documentation, "docs/src/ptf-catalog/sources"), - (Target::CDocumentation, "docs/src/reference/c"), - (Target::CppDocumentation, "docs/src/reference/cpp"), - (Target::PythonDocumentation, "docs/src/reference/python"), + for (layout, directory) in [ + (&DOCUMENTATION, "docs/src/ptf-catalog/sources"), + (&C_DOCUMENTATION, "docs/src/reference/c"), + (&CPP_DOCUMENTATION, "docs/src/reference/cpp"), + (&PYTHON_DOCUMENTATION, "docs/src/reference/python"), ] { assert_eq!( - target.output_path(root, relative), + root.join(layout.output_directory).join(relative), root.join(directory).join(relative) ); - assert_eq!(target.cleanup_directory(root), root.join(directory)); + assert_eq!(root.join(layout.cleanup_directory), root.join(directory)); } } + + #[test] + fn python_layouts_keep_extension_cleanup_scoped_to_its_package() { + assert_eq!(PYTHON_EXTENSION.output_directory, "targets/ptfkit-py"); + assert_eq!( + PYTHON_EXTENSION.cleanup_directory, + "targets/ptfkit-py/src/ptfkit" + ); + assert_eq!(PYTHON_WRAPPER.output_directory, "targets/ptfkit-py/src"); + assert_eq!( + PYTHON_WRAPPER.cleanup_directory, + "targets/ptfkit-py/src/ptfkit" + ); + } } diff --git a/codegen/src/targets/write.rs b/codegen/src/targets/write.rs index 1635e44..0f3f164 100644 --- a/codegen/src/targets/write.rs +++ b/codegen/src/targets/write.rs @@ -9,12 +9,12 @@ use std::{ use anyhow::{Context, Result, bail}; -use super::{Target, TargetOutput}; +use super::{Formatter, LAYOUTS, Layout, Output}; struct StagedWrite { target: PathBuf, temporary: PathBuf, - output_target: Target, + layout: &'static Layout, } static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); @@ -23,11 +23,11 @@ pub(super) struct GeneratedTree(BTreeMap>); pub(super) fn snapshot_generated(root: &Path) -> Result { let mut files = BTreeMap::new(); - for target in Target::ALL { + for layout in LAYOUTS { collect_generated( root, - &target.cleanup_directory(root), - target.generated_header(), + &root.join(layout.cleanup_directory), + layout.generated_header, &mut files, )?; } @@ -112,7 +112,7 @@ fn append_paths(report: &mut String, label: &str, paths: &[&PathBuf]) { } } -pub(super) fn commit(root: &Path, outputs: &[TargetOutput]) -> Result<()> { +pub(super) fn commit(root: &Path, outputs: &[Output]) -> Result<()> { let staged = stage(root, outputs)?; if let Err(error) = format(root, &staged) { remove_temporary(&staged); @@ -130,14 +130,14 @@ pub(super) fn commit(root: &Path, outputs: &[TargetOutput]) -> Result<()> { Ok(()) } -fn stage(root: &Path, outputs: &[TargetOutput]) -> Result> { +fn stage(root: &Path, outputs: &[Output]) -> Result> { let mut staged = Vec::new(); for output in outputs { for file in &output.files { - let target = output.target.output_path(root, &file.path); + let target = root.join(output.layout.output_directory).join(&file.path); if target.exists() && fs::read_to_string(&target)? == file.contents - && !output.target.is_clang_formatted() + && output.layout.formatter == Formatter::None { continue; } @@ -152,7 +152,7 @@ fn stage(root: &Path, outputs: &[TargetOutput]) -> Result> { staged.push(StagedWrite { target, temporary, - output_target: output.target, + layout: output.layout, }); } } @@ -182,16 +182,16 @@ fn write_temporary(path: &Path, contents: &str) -> Result<()> { .with_context(|| format!("writing temporary generated file {}", path.display())) } -fn cleanup(root: &Path, output: &TargetOutput) -> Result<()> { +fn cleanup(root: &Path, output: &Output) -> Result<()> { let expected = output .files .iter() - .map(|file| output.target.output_path(root, &file.path)) + .map(|file| root.join(output.layout.output_directory).join(&file.path)) .collect::>(); remove_obsolete( - &output.target.cleanup_directory(root), + &root.join(output.layout.cleanup_directory), &expected, - output.target.generated_header(), + output.layout.generated_header, ) } @@ -222,19 +222,29 @@ fn is_generated(contents: &[u8], header: &str) -> bool { } fn format(root: &Path, staged: &[StagedWrite]) -> Result<()> { - for target in Target::ALL { + for layout in LAYOUTS { let paths = staged .iter() - .filter(|file| file.output_target == target) + .filter(|file| std::ptr::eq(file.layout, layout)) .map(|file| file.temporary.clone()) .collect::>(); if !paths.is_empty() { - target.format(root, &paths)?; + format_layout(layout.formatter, root, &paths)?; } } Ok(()) } +fn format_layout(formatter: Formatter, root: &Path, paths: &[PathBuf]) -> Result<()> { + match formatter { + Formatter::None => Ok(()), + Formatter::Rust => format_rust(paths), + Formatter::Python => format_python(root, paths), + Formatter::C => format_c(paths), + Formatter::Cpp => format_cpp(paths), + } +} + pub(super) fn format_rust(paths: &[PathBuf]) -> Result<()> { run("rustfmt", &["--edition", "2024"], paths, None) } @@ -301,7 +311,7 @@ mod tests { use std::{collections::BTreeMap, fs, path::Path}; use super::*; - use crate::targets::{GeneratedFile, documentation::HEADER}; + use crate::targets::{GeneratedFile, Output, PYTHON_DOCUMENTATION, documentation::HEADER}; fn temporary_root(label: &str) -> PathBuf { std::env::temp_dir().join(format!( @@ -328,8 +338,8 @@ mod tests { ) .expect("write C page"); - let output = TargetOutput::new( - Target::PythonDocumentation, + let output = Output::new( + &PYTHON_DOCUMENTATION, vec![GeneratedFile::new( "index.md".into(), format!("{HEADER}# Python\n"), From d6afdcd61c07e1b1362fcc63a773a97840463110 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 23:36:24 +0300 Subject: [PATCH 09/14] refactor(codegen): align Rust output pipeline --- codegen/src/targets/mod.rs | 5 +---- codegen/src/targets/rs.rs | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/codegen/src/targets/mod.rs b/codegen/src/targets/mod.rs index ee59b6f..2c949da 100644 --- a/codegen/src/targets/mod.rs +++ b/codegen/src/targets/mod.rs @@ -198,10 +198,7 @@ pub(crate) fn run(root: &Path, entries: Vec) -> Result<()> { let documentation = documentation::render(&entries); let python_documentation = python_documentation::render(&entries); let compiled = compile::functions(entries)?; - let rust = rs::render(&compiled)? - .into_iter() - .map(|(path, contents)| GeneratedFile::new(path, contents)) - .collect::>(); + let rust = rs::render(&compiled)?; let py = py::render(&compiled)?; let native = native::render(&compiled)?; let c_documentation = c_documentation::render(&compiled)?; diff --git a/codegen/src/targets/rs.rs b/codegen/src/targets/rs.rs index c961b1a..bfa6072 100644 --- a/codegen/src/targets/rs.rs +++ b/codegen/src/targets/rs.rs @@ -10,11 +10,11 @@ use crate::{ semantic::{self, BinaryOp, Expr, MathFunction, Number, Reference, UnaryOp}, }; -use super::{documentation, group_by_source}; +use super::{GeneratedFile, documentation, group_by_source}; pub(super) const HEADER: &str = "// @generated by ptfkit-codegen; DO NOT EDIT.\n"; -pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { group_by_source(functions) .into_iter() .map(|(slug, functions)| { @@ -37,7 +37,7 @@ pub(crate) fn render(functions: &[CompiledFunction]) -> Result>>()?; - Ok(( + Ok(GeneratedFile::new( PathBuf::from(format!("{slug}.rs")), render_tokens(module_docs, quote!(#(#definitions)*)), )) From 1869f26fa8c133d4db5ffb6779be23954dda9d75 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Thu, 20 Aug 2026 23:38:45 +0300 Subject: [PATCH 10/14] refactor(codegen): converge output artifacts --- codegen/README.md | 30 +++++++++++++++++++++++++++ codegen/src/targets/mod.rs | 34 ++++--------------------------- codegen/src/targets/py/c.rs | 15 ++++++++++---- codegen/src/targets/py/mod.rs | 21 ++++++++++++------- codegen/src/targets/py/test.rs | 21 ++++++++++--------- codegen/src/targets/py/wrapper.rs | 18 +++++++++------- 6 files changed, 81 insertions(+), 58 deletions(-) create mode 100644 codegen/README.md diff --git a/codegen/README.md b/codegen/README.md new file mode 100644 index 0000000..086d6ca --- /dev/null +++ b/codegen/README.md @@ -0,0 +1,30 @@ +# Code Generator Architecture + +`ptfkit-codegen` validates the YAML specifications, compiles their formulas +into a shared semantic representation, and renders every committed target. +The generated files, after their target formatter runs, are the compatibility +contract; internal generator APIs are not. + +## Pipeline + +1. `specs` loads source specifications and `validate` checks their contracts. +2. `targets::compile` resolves formulas and golden cases into + `CompiledFunction` values. +3. Target renderers consume those values and return `GeneratedFile` artifacts. + `documentation` provides borrowed source/function facts to every renderer; + it deliberately contains no target markup. +4. `targets::run` assigns artifacts to declarative `Layout` values. +5. `targets::write` stages files, runs each layout's formatter, atomically + replaces targets, and removes obsolete marker-owned files. + +`check-generated` snapshots all marker-owned files, runs the same pipeline, +and fails if the generated tree changes. + +## Extension points + +Add target-local syntax and escaping beside its renderer. Use +`render::Writer` only where indentation-aware text composition is natural; +Rust remains token-based with `proc_macro2` and `quote`. Add a `Layout` only +when a new output group needs a distinct root, cleanup root, marker, or +formatter. Do not introduce a cross-language syntax AST or let renderers own +filesystem, staging, formatting, or cleanup policy. diff --git a/codegen/src/targets/mod.rs b/codegen/src/targets/mod.rs index 2c949da..eaa5096 100644 --- a/codegen/src/targets/mod.rs +++ b/codegen/src/targets/mod.rs @@ -16,7 +16,7 @@ use std::{ use anyhow::Result; -use crate::model::{CompiledFunction, Entry, PythonGeneration}; +use crate::model::{CompiledFunction, Entry}; pub(super) fn group_by_source( functions: &[CompiledFunction], @@ -203,32 +203,6 @@ pub(crate) fn run(root: &Path, entries: Vec) -> Result<()> { let native = native::render(&compiled)?; let c_documentation = c_documentation::render(&compiled)?; let cpp_documentation = cpp_documentation::render(&compiled)?; - let c = py - .c_sources - .into_iter() - .map(|(path, contents)| GeneratedFile::new(path.into(), contents)) - .collect::>(); - let mut wrappers = py - .wrappers - .into_iter() - .filter_map(|(module, mode, contents)| { - (mode == PythonGeneration::Generated).then_some(GeneratedFile::new( - PathBuf::from(module.replace('.', "/")).with_extension("py"), - contents, - )) - }) - .collect::>(); - wrappers.push(GeneratedFile::new("ptfkit/_ptfkit.pyi".into(), py.stub)); - let tests = py - .tests - .into_iter() - .filter_map(|(slug, mode, contents)| { - (mode == PythonGeneration::Generated).then_some(GeneratedFile::new( - format!("tests/test_{slug}.py").into(), - contents, - )) - }) - .collect::>(); write::commit( root, &[ @@ -237,9 +211,9 @@ pub(crate) fn run(root: &Path, entries: Vec) -> Result<()> { Output::new(&CPP_DOCUMENTATION, cpp_documentation), Output::new(&PYTHON_DOCUMENTATION, python_documentation), Output::new(&RUST, rust), - Output::new(&PYTHON_EXTENSION, c), - Output::new(&PYTHON_WRAPPER, wrappers), - Output::new(&PYTHON_TEST, tests), + Output::new(&PYTHON_EXTENSION, py.c_sources), + Output::new(&PYTHON_WRAPPER, py.wrappers), + Output::new(&PYTHON_TEST, py.tests), Output::new(&NATIVE_C, native.c_headers), Output::new(&NATIVE_CPP_MODULE, native.cpp_modules), Output::new(&NATIVE_CPP_CMAKE, native.cpp_cmake), diff --git a/codegen/src/targets/py/c.rs b/codegen/src/targets/py/c.rs index 35e64cb..82ac41c 100644 --- a/codegen/src/targets/py/c.rs +++ b/codegen/src/targets/py/c.rs @@ -5,13 +5,14 @@ use crate::render::Writer; use super::{ super::{ + GeneratedFile, c_expression::{self, Dialect}, group_by_source, }, C_HEADER, }; -pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { let mut includes = Writer::new(); let mut registers = Vec::new(); let mut writes = Vec::new(); @@ -41,7 +42,10 @@ pub(crate) fn render(functions: &[CompiledFunction]) -> Result Result, - pub(super) stub: String, - pub(super) tests: Vec<(String, PythonGeneration, String)>, - pub(super) wrappers: Vec<(String, PythonGeneration, String)>, + pub(super) c_sources: Vec, + pub(super) wrappers: Vec, + pub(super) tests: Vec, } pub(super) fn render(functions: &[CompiledFunction]) -> anyhow::Result { Ok(Output { c_sources: c::render(functions)?, - stub: stub::render(functions), + wrappers: { + let mut wrappers = wrapper::render(functions)?; + wrappers.push(GeneratedFile::new( + "ptfkit/_ptfkit.pyi".into(), + stub::render(functions), + )); + wrappers + }, tests: test::render(functions), - wrappers: wrapper::render(functions)?, }) } diff --git a/codegen/src/targets/py/test.rs b/codegen/src/targets/py/test.rs index 42422b7..a5a600d 100644 --- a/codegen/src/targets/py/test.rs +++ b/codegen/src/targets/py/test.rs @@ -2,9 +2,9 @@ use std::collections::BTreeMap; use crate::model::{CompiledFunction, Function, Outputs, PythonGeneration}; -use super::{WRAPPER_HEADER, natural_sort_key, syntax::Module}; +use super::{super::GeneratedFile, WRAPPER_HEADER, natural_sort_key, syntax::Module}; -pub(super) fn render(functions: &[CompiledFunction]) -> Vec<(String, PythonGeneration, String)> { +pub(super) fn render(functions: &[CompiledFunction]) -> Vec { let mut modules: BTreeMap> = BTreeMap::new(); for function in functions { modules @@ -15,14 +15,15 @@ pub(super) fn render(functions: &[CompiledFunction]) -> Vec<(String, PythonGener modules .into_iter() - .map(|(slug, functions)| { - let mode = functions[0].entry.spec.generation.public_python; - let source = if mode == PythonGeneration::Generated { - module_source(&slug, &functions) - } else { - String::new() - }; - (slug, mode, source) + .filter_map(|(slug, functions)| { + (functions[0].entry.spec.generation.public_python == PythonGeneration::Generated).then( + || { + GeneratedFile::new( + format!("tests/test_{slug}.py").into(), + module_source(&slug, &functions), + ) + }, + ) }) .collect() } diff --git a/codegen/src/targets/py/wrapper.rs b/codegen/src/targets/py/wrapper.rs index b1747df..82731ae 100644 --- a/codegen/src/targets/py/wrapper.rs +++ b/codegen/src/targets/py/wrapper.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, path::PathBuf}; use anyhow::{Result, bail}; @@ -7,7 +7,11 @@ use crate::{ model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}, }; -use super::{super::documentation, WRAPPER_HEADER, natural_sort_key, syntax::Module}; +use super::{ + super::{GeneratedFile, documentation}, + WRAPPER_HEADER, natural_sort_key, + syntax::Module, +}; struct PythonFunction<'a> { name: &'a str, @@ -26,9 +30,7 @@ struct PythonResultClass { docstring: String, } -pub(crate) fn render( - functions: &[CompiledFunction], -) -> Result> { +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { let mut modules: BTreeMap> = BTreeMap::new(); for function in functions { modules @@ -41,7 +43,6 @@ pub(crate) fn render( for (module, functions) in modules { let mode = functions[0].entry.spec.generation.public_python; if mode == PythonGeneration::Manual { - generated.push((module, mode, String::new())); continue; } let source = &functions[0].entry.spec.source; @@ -94,7 +95,10 @@ pub(crate) fn render( &functions, &exports, ); - generated.push((module, mode, text)); + generated.push(GeneratedFile::new( + PathBuf::from(module.replace('.', "/")).with_extension("py"), + text, + )); } Ok(generated) } From 82540bf11d6e5beb8dc25596f3324a9171db3443 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Fri, 21 Aug 2026 08:58:48 +0300 Subject: [PATCH 11/14] refactor(codegen): structure documentation rendering --- codegen/src/render.rs | 35 -- codegen/src/targets/c_documentation.rs | 259 ++++++++------ codegen/src/targets/cpp_documentation.rs | 250 ++++++++----- codegen/src/targets/documentation.rs | 375 +++++++++++++------- codegen/src/targets/python_documentation.rs | 82 +++-- 5 files changed, 605 insertions(+), 396 deletions(-) diff --git a/codegen/src/render.rs b/codegen/src/render.rs index d872eb4..93c3527 100644 --- a/codegen/src/render.rs +++ b/codegen/src/render.rs @@ -44,22 +44,6 @@ impl Writer { self.indentation -= 1; } - pub(crate) fn block( - &mut self, - opening: impl Display, - indent_contents: bool, - render: impl FnOnce(&mut Self), - closing: impl Display, - ) { - self.line(opening); - if indent_contents { - self.indented(render); - } else { - render(self); - } - self.line(closing); - } - pub(crate) fn into_string(self) -> String { self.contents } @@ -103,25 +87,6 @@ mod tests { assert_eq!(writer.into_string(), "value: 42\n\nnext\n"); } - #[test] - fn renders_nested_blocks_with_indentation() { - let mut writer = Writer::new(); - writer.block( - "outer {", - true, - |writer| { - writer.line("first;"); - writer.block("inner {", true, |writer| writer.line("second;"), "}"); - }, - "}", - ); - - assert_eq!( - writer.into_string(), - "outer {\n first;\n inner {\n second;\n }\n}\n" - ); - } - #[test] fn composes_render_values() { let mut writer = Writer::new(); diff --git a/codegen/src/targets/c_documentation.rs b/codegen/src/targets/c_documentation.rs index 45ba474..e729aff 100644 --- a/codegen/src/targets/c_documentation.rs +++ b/codegen/src/targets/c_documentation.rs @@ -5,58 +5,73 @@ use anyhow::Result; use crate::{ documentation::{self as docs}, model::{CompiledFunction, Output, Parameter}, + render::Writer, }; use super::{ GeneratedFile, - documentation::{FRONTMATTER_HEADER, HEADER}, + documentation::{self as markdown, HEADER}, group_by_source, native::c_result_name, }; pub(super) fn render(functions: &[CompiledFunction]) -> Result> { let sources = group_by_source(functions); - let mut files = vec![file("index.md", index(&sources))]; - files.push(header_file( - "headers/ptfkit.md", - "ptfkit", - umbrella(&sources), - )); + let mut files = vec![markdown::markdown_file("index.md", |writer| { + render_index(writer, &sources); + })]; + files.push(header_file("headers/ptfkit.md", "ptfkit", |writer| { + render_umbrella(writer, &sources); + Ok(()) + })?); let mut index_entries = Vec::new(); for (slug, functions) in sources { - files.push(header_file( - format!("headers/{slug}.md"), - slug, - header(slug, &functions)?, - )); + files.push(header_file(format!("headers/{slug}.md"), slug, |writer| { + render_header(writer, slug, &functions) + })?); for function in functions { index_entries.push((slug, function)); } } index_entries.sort_by_key(|(_, function)| natural_sort_key(&function.core.name)); - files.push(file("functions.md", functions_index(&index_entries))); + files.push(markdown::markdown_file("functions.md", |writer| { + render_functions_index(writer, &index_entries); + })); Ok(files) } -fn file(path: impl Into, contents: String) -> GeneratedFile { - GeneratedFile::new(path.into(), format!("{}\n", contents.trim_end())) -} - -fn header_file(path: impl Into, slug: &str, contents: String) -> GeneratedFile { - GeneratedFile::new( +fn header_file( + path: impl Into, + slug: &str, + render: impl FnOnce(&mut Writer) -> Result<()>, +) -> Result { + let mut writer = Writer::new(); + markdown::frontmatter(&mut writer, |writer| { + writer.line(format_args!("title: \"{slug}.h\"")); + }); + render(&mut writer)?; + Ok(GeneratedFile::new( path.into(), - format!("---\ntitle: \"{slug}.h\"\n---\n\n{}\n", contents.trim_end()), - ) + markdown::markdown_contents(writer), + )) } -fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C API reference\n---\n\n"); - text.push_str("# C API reference\n\n"); - text.push_str("ptfkit's C API is organized around installed headers.\n\n"); - text.push_str("## Headers\n\n"); - text.push_str( - "- [``](headers/ptfkit.md) — Aggregates every ptfkit source header.\n", +fn render_index( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C API reference"); + }); + writer.line("# C API reference"); + writer.blank_line(); + writer.line("ptfkit's C API is organized around installed headers."); + writer.blank_line(); + writer.line("## Headers"); + writer.blank_line(); + writer.line( + "- [``](headers/ptfkit.md) — Aggregates every ptfkit source header.", ); for (slug, functions) in sources { let summary = docs::for_source( @@ -64,24 +79,32 @@ fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> &functions[0].entry.spec.scope, ) .summary; - text.push_str(&format!( - "- [``](headers/{slug}.md) — {}\n", + writer.line(format_args!( + "- [``](headers/{slug}.md) — {}", escape_text(summary) )); } - text.push_str("\nSee the [function index](functions.md) for all public C functions.\n"); - text + writer.blank_line(); + writer.line("See the [function index](functions.md) for all public C functions."); } -fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = String::from(HEADER); - text.push_str("# ``\n\n"); - text.push_str("```c\n#include \n```\n\n"); - text.push_str("This umbrella header aggregates every public ptfkit source header. Include an individual header when only one source is needed.\n\n"); - text.push_str("## Included headers\n\n"); +fn render_umbrella( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + writer.write(HEADER); + writer.line("# ``"); + writer.blank_line(); + markdown::code_block(writer, "c", |writer| { + writer.line("#include "); + }); + writer.line("This umbrella header aggregates every public ptfkit source header. Include an individual header when only one source is needed."); + writer.blank_line(); + writer.line("## Included headers"); + writer.blank_line(); for (slug, functions) in sources { - text.push_str(&format!( - "- [``]({slug}.md) — {}\n", + writer.line(format_args!( + "- [``]({slug}.md) — {}", escape_text( docs::for_source( &functions[0].entry.spec.source, @@ -91,40 +114,49 @@ fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) ) )); } - text } -fn header(slug: &str, functions: &[&CompiledFunction]) -> Result { +fn render_header(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction]) -> Result<()> { let first = functions .first() .expect("compiled source contains at least one function"); let source = docs::for_source(&first.entry.spec.source, &first.entry.spec.scope); - let mut text = String::from(HEADER); - text.push_str(&format!("# ``\n\n")); - text.push_str(&format!("```c\n#include \n```\n\n")); - text.push_str(&format!("{}\n\n", escape_text(source.summary))); - text.push_str("## Source\n\n"); - text.push_str(&escape_text(source.reference.citation)); - text.push_str("\n\n"); + writer.write(HEADER); + writer.line(format_args!("# ``")); + writer.blank_line(); + markdown::code_block(writer, "c", |writer| { + writer.line(format_args!("#include ")); + }); + writer.line(escape_text(source.summary)); + writer.blank_line(); + writer.line("## Source"); + writer.blank_line(); + writer.line(escape_text(source.reference.citation)); + writer.blank_line(); if let Some(doi) = source.reference.doi { - text.push_str(&format!( - "[DOI: {}]({})\n\n", + writer.line(format_args!( + "[DOI: {}]({})", escape_text(doi.identifier), doi.url )); + writer.blank_line(); } if source.territory.is_some() || source.dataset.is_some() { - text.push_str("## Scope\n\n"); + writer.line("## Scope"); + writer.blank_line(); if let Some(territory) = source.territory { - text.push_str(&format!("**Territory:** {}\n\n", escape_text(territory))); + writer.line(format_args!("**Territory:** {}", escape_text(territory))); + writer.blank_line(); } if let Some(dataset) = source.dataset { - text.push_str(&format!("**Dataset:** {}\n\n", escape_text(dataset))); + writer.line(format_args!("**Dataset:** {}", escape_text(dataset))); + writer.blank_line(); } } - text.push_str(&format!( - "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" + writer.line(format_args!( + "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)" )); + writer.blank_line(); let mut structures = BTreeSet::new(); for function in functions { @@ -135,88 +167,107 @@ fn header(slug: &str, functions: &[&CompiledFunction]) -> Result { .expect("record output has a result class"); let c_name = c_result_name(name); if structures.insert(c_name.clone()) { - text.push_str(&structure(&c_name, spec.outputs.fields())); + render_structure(writer, &c_name, spec.outputs.fields()); } } } - text.push_str("## Functions\n\n"); + writer.line("## Functions"); + writer.blank_line(); for function in functions { - text.push_str(&function_documentation(function)?); + render_function_documentation(writer, function)?; } - Ok(text) + Ok(()) } -fn structure(name: &str, fields: &[Parameter]) -> String { - let mut text = format!("## `{name}`\n\n"); - text.push_str("```c\n"); - text.push_str(&format!( - "typedef struct {{\n{} }} {name};\n", - fields - .iter() - .map(|field| format!(" double {};\n", field.name)) - .collect::() - )); - text.push_str("```\n\n| Field | Description |\n| --- | --- |\n"); +fn render_structure(writer: &mut Writer, name: &str, fields: &[Parameter]) { + writer.line(format_args!("## `{name}`")); + writer.blank_line(); + markdown::code_block(writer, "c", |writer| { + writer.line("typedef struct {"); + writer.indented(|writer| { + for field in fields { + writer.line(format_args!("double {};", field.name)); + } + }); + writer.line(format_args!(" }} {name};")); + }); + writer.line("| Field | Description |"); + writer.line("| --- | --- |"); for field in fields { - text.push_str(&format!( - "| `{}` | {} |\n", + writer.line(format_args!( + "| `{}` | {} |", field.name, parameter_details(field) )); } - text.push('\n'); - text + writer.blank_line(); } -fn function_documentation(function: &CompiledFunction) -> Result { +fn render_function_documentation(writer: &mut Writer, function: &CompiledFunction) -> Result<()> { let spec = spec(function); let document = docs::for_function(spec); let anchor = function_anchor(&function.core.name); - let mut text = format!("### `{}` {{#{anchor}}}\n\n", function.core.name); - text.push_str(&format!("{}\n\n", escape_text(document.summary))); - text.push_str("```c\n"); - text.push_str(&signature(function)?); - text.push_str(";\n```\n\n"); - text.push_str("#### Parameters\n\n| Name | Direction | Description |\n| --- | --- | --- |\n"); + writer.line(format_args!("### `{}` {{#{anchor}}}", function.core.name)); + writer.blank_line(); + writer.line(escape_text(document.summary)); + writer.blank_line(); + let signature = signature(function)?; + markdown::code_block(writer, "c", |writer| { + writer.line(format_args!("{signature};")); + }); + writer.line("#### Parameters"); + writer.blank_line(); + writer.line("| Name | Direction | Description |"); + writer.line("| --- | --- | --- |"); for parameter in document.parameters { - text.push_str(&format!( - "| `{}` | in | {} |\n", + writer.line(format_args!( + "| `{}` | in | {} |", parameter.name, parameter_details(parameter) )); } - text.push_str("\n#### Returns\n\n"); + writer.blank_line(); + writer.line("#### Returns"); + writer.blank_line(); match document.returns { - docs::Returns::Scalar(field) => text.push_str(&format!("{}\n\n", parameter_details(field))), + docs::Returns::Scalar(field) => { + writer.line(parameter_details(field)); + writer.blank_line(); + } docs::Returns::Record { .. } => { let name = spec .result_class() .expect("record output has a result class"); - text.push_str(&format!("A `{}` value.\n\n", c_result_name(name))); + writer.line(format_args!("A `{}` value.", c_result_name(name))); + writer.blank_line(); } } for note in document.notes { - admonition(&mut text, "note", note); + render_admonition(writer, "note", note); } for warning in document.warnings { - admonition(&mut text, "warning", warning); + render_admonition(writer, "warning", warning); } - Ok(text) + Ok(()) } -fn functions_index(functions: &[(&str, &CompiledFunction)]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C function index\n---\n\n"); - text.push_str("# C function index\n\n| Function | Summary | Header |\n| --- | --- | --- |\n"); +fn render_functions_index(writer: &mut Writer, functions: &[(&str, &CompiledFunction)]) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C function index"); + }); + writer.line("# C function index"); + writer.blank_line(); + writer.line("| Function | Summary | Header |"); + writer.line("| --- | --- | --- |"); for (slug, function) in functions { - text.push_str(&format!( - "| [`{0}`](headers/{1}.md#{2}) | {3} | [``](headers/{1}.md) |\n", + writer.line(format_args!( + "| [`{0}`](headers/{1}.md#{2}) | {3} | [``](headers/{1}.md) |", function.core.name, slug, function_anchor(&function.core.name), escape_table(docs::for_function(spec(function)).summary), )); } - text } fn signature(function: &CompiledFunction) -> Result { @@ -257,12 +308,8 @@ fn parameter_details(parameter: &Parameter) -> String { ) } -fn admonition(text: &mut String, kind: &str, body: &str) { - text.push_str(&format!("!!! {kind}\n\n")); - for line in body.lines() { - text.push_str(&format!(" {}\n", escape_text(line))); - } - text.push('\n'); +fn render_admonition(writer: &mut Writer, kind: &str, body: &str) { + markdown::admonition(writer, kind, body, escape_text); } fn escape_text(value: &str) -> String { @@ -364,11 +411,11 @@ mod tests { #[test] fn renders_notes_and_warnings_as_admonitions() { - let mut text = String::new(); - admonition(&mut text, "note", "Keep `value` in range."); - admonition(&mut text, "warning", "Do not use | as a separator."); + let mut writer = Writer::new(); + render_admonition(&mut writer, "note", "Keep `value` in range."); + render_admonition(&mut writer, "warning", "Do not use | as a separator."); assert_eq!( - text, + writer.into_string(), "!!! note\n\n Keep \\`value\\` in range.\n\n!!! warning\n\n Do not use | as a separator.\n\n" ); } diff --git a/codegen/src/targets/cpp_documentation.rs b/codegen/src/targets/cpp_documentation.rs index 99137c8..b9007b1 100644 --- a/codegen/src/targets/cpp_documentation.rs +++ b/codegen/src/targets/cpp_documentation.rs @@ -5,46 +5,64 @@ use anyhow::{Result, anyhow}; use crate::{ documentation::{self as docs}, model::{CompiledFunction, Output, Parameter}, + render::Writer, }; -use super::{GeneratedFile, documentation::FRONTMATTER_HEADER, group_by_source}; +use super::{ + GeneratedFile, + documentation::{self as markdown}, + group_by_source, +}; pub(super) fn render(functions: &[CompiledFunction]) -> Result> { let sources = group_by_source(functions); - let mut files = vec![file("index.md", index(&sources))]; - files.push(GeneratedFile::new( - "modules/ptfkit.md".into(), - umbrella(&sources), - )); + let mut files = vec![markdown::markdown_file("index.md", |writer| { + render_index(writer, &sources); + })]; + files.push(markdown::markdown_file("modules/ptfkit.md", |writer| { + render_umbrella(writer, &sources); + })); let mut index_entries = Vec::new(); for (slug, functions) in sources { - files.push(GeneratedFile::new( - format!("modules/{slug}.md").into(), - module(slug, &functions)?, - )); + files.push(module_file(slug, &functions)?); for function in functions { index_entries.push((slug, function)); } } index_entries.sort_by_key(|(_, function)| natural_sort_key(&function.core.name)); - files.push(file("functions.md", functions_index(&index_entries))); + files.push(markdown::markdown_file("functions.md", |writer| { + render_functions_index(writer, &index_entries); + })); Ok(files) } -fn file(path: impl Into, contents: String) -> GeneratedFile { - GeneratedFile::new(path.into(), format!("{}\n", contents.trim_end())) +fn module_file(slug: &str, functions: &[&CompiledFunction]) -> Result { + let mut writer = Writer::new(); + render_module(&mut writer, slug, functions)?; + Ok(GeneratedFile::new( + PathBuf::from(format!("modules/{slug}.md")), + markdown::markdown_contents(writer), + )) } -fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C++ API reference\n---\n\n"); - text.push_str("# C++ API reference\n\n"); - text.push_str("ptfkit's C++ API is organized around C++20 modules.\n\n"); - text.push_str("## Modules\n\n"); - text.push_str("- [`ptfkit`](modules/ptfkit.md) — Re-exports every ptfkit source module.\n"); +fn render_index( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C++ API reference"); + }); + writer.line("# C++ API reference"); + writer.blank_line(); + writer.line("ptfkit's C++ API is organized around C++20 modules."); + writer.blank_line(); + writer.line("## Modules"); + writer.blank_line(); + writer.line("- [`ptfkit`](modules/ptfkit.md) — Re-exports every ptfkit source module."); for (slug, functions) in sources { - text.push_str(&format!( - "- [`ptfkit.{slug}`](modules/{slug}.md) — {}\n", + writer.line(format_args!( + "- [`ptfkit.{slug}`](modules/{slug}.md) — {}", escape_text( docs::for_source( &functions[0].entry.spec.source, @@ -54,19 +72,30 @@ fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> ) )); } - text.push_str("\nSee the [function index](functions.md) for all public C++ functions.\n"); - text + writer.blank_line(); + writer.line("See the [function index](functions.md) for all public C++ functions."); } -fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = - format!("---\n{FRONTMATTER_HEADER}title: C++ module ptfkit\nnav-title: ptfkit\n---\n\n"); - text.push_str("# `ptfkit`\n\n```cpp\nimport ptfkit;\n```\n\n"); - text.push_str("This umbrella module re-exports every public ptfkit source module. Import an individual module when only one source is needed.\n\n"); - text.push_str("## Re-exported modules\n\n"); +fn render_umbrella( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C++ module ptfkit"); + writer.line("nav-title: ptfkit"); + }); + writer.line("# `ptfkit`"); + writer.blank_line(); + markdown::code_block(writer, "cpp", |writer| { + writer.line("import ptfkit;"); + }); + writer.line("This umbrella module re-exports every public ptfkit source module. Import an individual module when only one source is needed."); + writer.blank_line(); + writer.line("## Re-exported modules"); + writer.blank_line(); for (slug, functions) in sources { - text.push_str(&format!( - "- [`ptfkit.{slug}`]({slug}.md) — {}\n", + writer.line(format_args!( + "- [`ptfkit.{slug}`]({slug}.md) — {}", escape_text( docs::for_source( &functions[0].entry.spec.source, @@ -76,124 +105,157 @@ fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) ) )); } - text } -fn module(slug: &str, functions: &[&CompiledFunction]) -> Result { +fn render_module(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction]) -> Result<()> { let first = functions .first() .expect("compiled source contains at least one function"); let source = docs::for_source(&first.entry.spec.source, &first.entry.spec.scope); - let mut text = format!( - "---\n{FRONTMATTER_HEADER}title: C++ module ptfkit.{slug}\nnav-title: ptfkit.{slug}\n---\n\n" - ); - text.push_str(&format!( - "# `ptfkit.{slug}`\n\n```cpp\nimport ptfkit.{slug};\n```\n\n" - )); - text.push_str(&format!("**Exported namespace:** `ptfkit::{slug}`\n\n")); - text.push_str(&format!("{}\n\n", escape_text(source.summary))); - text.push_str("## Source\n\n"); - text.push_str(&escape_text(source.reference.citation)); - text.push_str("\n\n"); + markdown::generated_frontmatter(writer, |writer| { + writer.line(format_args!("title: C++ module ptfkit.{slug}")); + writer.line(format_args!("nav-title: ptfkit.{slug}")); + }); + writer.line(format_args!("# `ptfkit.{slug}`")); + writer.blank_line(); + markdown::code_block(writer, "cpp", |writer| { + writer.line(format_args!("import ptfkit.{slug};")); + }); + writer.line(format_args!("**Exported namespace:** `ptfkit::{slug}`")); + writer.blank_line(); + writer.line(escape_text(source.summary)); + writer.blank_line(); + writer.line("## Source"); + writer.blank_line(); + writer.line(escape_text(source.reference.citation)); + writer.blank_line(); if let Some(doi) = source.reference.doi { - text.push_str(&format!( - "[DOI: {}]({})\n\n", + writer.line(format_args!( + "[DOI: {}]({})", escape_text(doi.identifier), doi.url )); + writer.blank_line(); } if source.territory.is_some() || source.dataset.is_some() { - text.push_str("## Scope\n\n"); + writer.line("## Scope"); + writer.blank_line(); if let Some(territory) = source.territory { - text.push_str(&format!("**Territory:** {}\n\n", escape_text(territory))); + writer.line(format_args!("**Territory:** {}", escape_text(territory))); + writer.blank_line(); } if let Some(dataset) = source.dataset { - text.push_str(&format!("**Dataset:** {}\n\n", escape_text(dataset))); + writer.line(format_args!("**Dataset:** {}", escape_text(dataset))); + writer.blank_line(); } } - text.push_str(&format!( - "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" + writer.line(format_args!( + "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)" )); + writer.blank_line(); let mut structures = BTreeSet::new(); for function in functions { if let Output::Struct(_) = &function.core.output { let result = result_class(function)?; if structures.insert(result) { - text.push_str(&structure(result, spec(function).outputs.fields())); + render_structure(writer, result, spec(function).outputs.fields()); } } } - text.push_str("## Functions\n\n"); + writer.line("## Functions"); + writer.blank_line(); for function in functions { - text.push_str(&function_documentation(function)?); + render_function_documentation(writer, function)?; } - text.pop(); - Ok(text) + Ok(()) } -fn structure(name: &str, fields: &[Parameter]) -> String { - let mut text = format!("## `{name}`\n\n```cpp\nstruct {name} {{\n"); - for field in fields { - text.push_str(&format!(" double {};\n", field.name)); - } - text.push_str("};\n```\n\n| Field | Description |\n| --- | --- |\n"); +fn render_structure(writer: &mut Writer, name: &str, fields: &[Parameter]) { + writer.line(format_args!("## `{name}`")); + writer.blank_line(); + markdown::code_block(writer, "cpp", |writer| { + writer.line(format_args!("struct {name} {{")); + writer.indented(|writer| { + for field in fields { + writer.line(format_args!("double {};", field.name)); + } + }); + writer.line("};"); + }); + writer.line("| Field | Description |"); + writer.line("| --- | --- |"); for field in fields { - text.push_str(&format!( - "| `{}` | {} |\n", + writer.line(format_args!( + "| `{}` | {} |", field.name, parameter_details(field) )); } - text.push('\n'); - text + writer.blank_line(); } -fn function_documentation(function: &CompiledFunction) -> Result { +fn render_function_documentation(writer: &mut Writer, function: &CompiledFunction) -> Result<()> { let spec = spec(function); let document = docs::for_function(spec); let anchor = function_anchor(&function.core.name); - let mut text = format!("### `{}` {{#{anchor}}}\n\n", function.core.name); - text.push_str(&format!("{}\n\n", escape_text(document.summary))); - text.push_str("```cpp\n"); - text.push_str(&signature(function)?); - text.push_str("\n```\n\n"); - text.push_str("#### Parameters\n\n| Name | Description |\n| --- | --- |\n"); + writer.line(format_args!("### `{}` {{#{anchor}}}", function.core.name)); + writer.blank_line(); + writer.line(escape_text(document.summary)); + writer.blank_line(); + let signature = signature(function)?; + markdown::code_block(writer, "cpp", |writer| { + writer.line(signature); + }); + writer.line("#### Parameters"); + writer.blank_line(); + writer.line("| Name | Description |"); + writer.line("| --- | --- |"); for parameter in document.parameters { - text.push_str(&format!( - "| `{}` | {} |\n", + writer.line(format_args!( + "| `{}` | {} |", parameter.name, parameter_details(parameter) )); } - text.push_str("\n#### Returns\n\n"); + writer.blank_line(); + writer.line("#### Returns"); + writer.blank_line(); match document.returns { - docs::Returns::Scalar(field) => text.push_str(&format!("{}\n\n", parameter_details(field))), + docs::Returns::Scalar(field) => { + writer.line(parameter_details(field)); + writer.blank_line(); + } docs::Returns::Record { .. } => { - text.push_str(&format!("A `{}` value.\n\n", result_class(function)?)) + writer.line(format_args!("A `{}` value.", result_class(function)?)); + writer.blank_line(); } } for note in document.notes { - admonition(&mut text, "note", note); + render_admonition(writer, "note", note); } for warning in document.warnings { - admonition(&mut text, "warning", warning); + render_admonition(writer, "warning", warning); } - Ok(text) + Ok(()) } -fn functions_index(functions: &[(&str, &CompiledFunction)]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C++ function index\n---\n\n"); - text.push_str("# C++ function index\n\n| Function | Summary | Module |\n| --- | --- | --- |\n"); +fn render_functions_index(writer: &mut Writer, functions: &[(&str, &CompiledFunction)]) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C++ function index"); + }); + writer.line("# C++ function index"); + writer.blank_line(); + writer.line("| Function | Summary | Module |"); + writer.line("| --- | --- | --- |"); for (slug, function) in functions { let qualified = format!("ptfkit::{slug}::{}", function.core.name); - text.push_str(&format!( - "| [`{qualified}`](modules/{slug}.md#{}) | {} | [`ptfkit.{slug}`](modules/{slug}.md) |\n", + writer.line(format_args!( + "| [`{qualified}`](modules/{slug}.md#{}) | {} | [`ptfkit.{slug}`](modules/{slug}.md) |", function_anchor(&function.core.name), escape_table(docs::for_function(spec(function)).summary), )); } - text } fn signature(function: &CompiledFunction) -> Result { @@ -236,12 +298,8 @@ fn parameter_details(parameter: &Parameter) -> String { ) } -fn admonition(text: &mut String, kind: &str, body: &str) { - text.push_str(&format!("!!! {kind}\n\n")); - for line in body.lines() { - text.push_str(&format!(" {}\n", escape_text(line))); - } - text.push('\n'); +fn render_admonition(writer: &mut Writer, kind: &str, body: &str) { + markdown::admonition(writer, kind, body, escape_text); } fn escape_text(value: &str) -> String { @@ -320,7 +378,11 @@ mod tests { .count(); assert_eq!(umbrella.matches("- [`ptfkit.").count(), source_modules); - assert!(index.find("[`ptfkit`](").unwrap() < index.find("ptfkit.ahuja1984").unwrap()); + assert!(index.find("[`ptfkit`]( ").is_none()); + assert!( + index.find("[`ptfkit`](modules/ptfkit.md)").unwrap() + < index.find("ptfkit.ahuja1984").unwrap() + ); assert!( umbrella.find("ptfkit.ahuja1984").unwrap() < umbrella.find("ptfkit.aimrun2009").unwrap() diff --git a/codegen/src/targets/documentation.rs b/codegen/src/targets/documentation.rs index 2318648..9b0024b 100644 --- a/codegen/src/targets/documentation.rs +++ b/codegen/src/targets/documentation.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use crate::{ documentation::{self as docs, FunctionDocument}, model::{Entry, Parameter}, + render::{Render, Writer}, }; pub(super) const HEADER: &str = "\n\n"; @@ -16,145 +17,281 @@ pub(super) fn parameter_documentation(parameter: &Parameter) -> String { format!("{}: {}", parameter.name, parameter_details(parameter)) } +pub(super) fn markdown_file( + path: impl Into, + render: impl FnOnce(&mut Writer), +) -> super::GeneratedFile { + let mut writer = Writer::new(); + render(&mut writer); + super::GeneratedFile::new(path.into(), markdown_contents(writer)) +} + +pub(super) fn markdown_contents(writer: Writer) -> String { + let contents = writer.into_string(); + format!("{}\n", contents.trim_end()) +} + +pub(super) fn frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { + writer.line("---"); + render(writer); + writer.line("---"); + writer.blank_line(); +} + +pub(super) fn generated_frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { + frontmatter(writer, |writer| { + writer.write(FRONTMATTER_HEADER); + render(writer); + }); +} + +pub(super) fn code_block(writer: &mut Writer, language: &str, render: impl FnOnce(&mut Writer)) { + writer.line(format_args!("```{language}")); + render(writer); + writer.line("```"); + writer.blank_line(); +} + +pub(super) fn admonition( + writer: &mut Writer, + kind: &str, + body: &str, + render_line: impl Fn(&str) -> String, +) { + writer.line(format_args!("!!! {kind}")); + writer.blank_line(); + writer.indented(|writer| { + for line in body.lines() { + writer.line(render_line(line)); + } + }); + writer.blank_line(); +} + pub(super) fn render(entries: &[Entry]) -> Vec { - let mut files = vec![super::GeneratedFile::new( - PathBuf::from("index.md"), - format!("{}\n", render_index(entries).trim_end()), - )]; + let mut files = vec![markdown_file("index.md", |writer| { + IndexPage { entries }.render(writer); + })]; for entry in entries { - files.push(super::GeneratedFile::new( - PathBuf::from(format!("{}.md", entry.slug)), - format!("{}\n", render_source(entry).trim_end()), - )); + files.push(markdown_file(format!("{}.md", entry.slug), |writer| { + SourcePage { entry }.render(writer); + })); } files } -fn render_index(entries: &[Entry]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: PTF Sources\n---\n\n"); - text.push_str("# PTF Sources\n\n"); - text.push_str( - "Each page describes the source, scope, inputs, outputs, status, and limitations of the \ - functions defined by one specification.\n\n", - ); - text.push_str("| Source | Territory | Functions |\n| --- | --- | ---: |\n"); - for entry in entries { - let source = docs::for_source(&entry.spec.source, &entry.spec.scope); - let territory = source.territory.unwrap_or("—"); - text.push_str(&format!( - "| [{}](./{}.md) | {} | {} |\n", - escape_table(source.summary), - entry.slug, - escape_table(territory), - entry.spec.functions.len(), - )); - } - text -} - -fn render_source(entry: &Entry) -> String { - let spec = &entry.spec; - let source = docs::for_source(&spec.source, &spec.scope); - let mut text = format!( - "---\n{FRONTMATTER_HEADER}title: PTF source {slug}\nnav-title: {slug}\n---\n\n", - slug = entry.slug - ); - text.push_str(&format!("# {}\n\n", source.summary)); - text.push_str("## Source\n\n"); - text.push_str(source.reference.citation); - text.push_str("\n\n"); - if let Some(doi) = source.reference.doi { - text.push_str(&format!("[DOI: {}]({})\n\n", doi.identifier, doi.url)); +struct IndexPage<'a> { + entries: &'a [Entry], +} + +impl Render for IndexPage<'_> { + fn render(&self, writer: &mut Writer) { + generated_frontmatter(writer, |writer| writer.line("title: PTF Sources")); + writer.line("# PTF Sources"); + writer.blank_line(); + writer.line( + "Each page describes the source, scope, inputs, outputs, status, and limitations of the functions defined by one specification.", + ); + writer.blank_line(); + writer.line("| Source | Territory | Functions |"); + writer.line("| --- | --- | ---: |"); + for entry in self.entries { + let source = docs::for_source(&entry.spec.source, &entry.spec.scope); + let territory = source.territory.unwrap_or("\u{2014}"); + writer.line(format_args!( + "| [{}](./{}.md) | {} | {} |", + escape_table(source.summary), + entry.slug, + escape_table(territory), + entry.spec.functions.len(), + )); + } } - if source.territory.is_some() || source.dataset.is_some() { - text.push_str("## Scope\n\n"); - if let Some(territory) = source.territory { - text.push_str(&format!("**Territory:** {territory}\n\n")); +} + +struct SourcePage<'a> { + entry: &'a Entry, +} + +impl Render for SourcePage<'_> { + fn render(&self, writer: &mut Writer) { + let spec = &self.entry.spec; + let source = docs::for_source(&spec.source, &spec.scope); + generated_frontmatter(writer, |writer| { + writer.line(format_args!("title: PTF source {}", self.entry.slug)); + writer.line(format_args!("nav-title: {}", self.entry.slug)); + }); + writer.line(format_args!("# {}", source.summary)); + writer.blank_line(); + writer.line("## Source"); + writer.blank_line(); + writer.line(source.reference.citation); + writer.blank_line(); + if let Some(doi) = source.reference.doi { + writer.line(format_args!("[DOI: {}]({})", doi.identifier, doi.url)); + writer.blank_line(); } - if let Some(dataset) = source.dataset { - text.push_str(&format!("**Dataset:** {dataset}\n\n")); + if source.territory.is_some() || source.dataset.is_some() { + writer.line("## Scope"); + writer.blank_line(); + if let Some(territory) = source.territory { + writer.line(format_args!("**Territory:** {territory}")); + writer.blank_line(); + } + if let Some(dataset) = source.dataset { + writer.line(format_args!("**Dataset:** {dataset}")); + writer.blank_line(); + } + } + writer.line("## Functions"); + writer.blank_line(); + for function in &spec.functions { + FunctionSection { + document: docs::for_function(function), + name: &function.public_api.name, + status: &function.status, + } + .render(writer); } } - text.push_str("## Functions\n\n"); - for function in &spec.functions { - render_function( - &mut text, - docs::for_function(function), - &function.public_api.name, - &function.status, - ); - } - text -} - -fn render_function(text: &mut String, document: FunctionDocument<'_>, name: &str, status: &str) { - text.push_str(&format!("### `{name}`\n\n")); - text.push_str(document.summary); - text.push_str("\n\n"); - text.push_str(&format!("**Status:** `{status}`\n\n")); - text.push_str(&format!( - "**Prediction target:** {}\n\n", - document.remarks.prediction_target - )); - - let models = [ - document - .models - .h_theta - .map(|model| format!("$h(\\theta)$ — {model}")), - document.models.k_h.map(|model| format!("$k(h)$ — {model}")), - ] - .into_iter() - .flatten() - .collect::>(); - if !models.is_empty() { - text.push_str(&format!("**Models:** {}\n\n", models.join("; "))); - } +} - render_parameters(text, "Inputs", document.parameters); - render_parameters( - text, - "Outputs", - match document.returns { - docs::Returns::Scalar(field) => std::slice::from_ref(field), - docs::Returns::Record { fields, .. } => fields, - }, - ); - for note in document.notes { - render_admonition(text, "note", note); - } - for warning in document.warnings { - render_admonition(text, "warning", warning); - } +struct FunctionSection<'a> { + document: FunctionDocument<'a>, + name: &'a str, + status: &'a str, } -fn render_parameters(text: &mut String, title: &str, parameters: &[Parameter]) { - text.push_str(&format!("#### {title}\n\n")); - text.push_str("| Name | Unit | Domain | Description |\n| --- | --- | --- | --- |\n"); - for parameter in parameters { - text.push_str(&format!( - "| `{}` | {} | {} | {} |\n", - parameter.name, - escape_table(¶meter.unit), - parameter - .domain - .as_deref() - .map(escape_table) - .unwrap_or_else(|| "—".into()), - escape_table(¶meter.description), +impl Render for FunctionSection<'_> { + fn render(&self, writer: &mut Writer) { + writer.line(format_args!("### `{}`", self.name)); + writer.blank_line(); + writer.line(self.document.summary); + writer.blank_line(); + writer.line(format_args!("**Status:** `{}`", self.status)); + writer.blank_line(); + writer.line(format_args!( + "**Prediction target:** {}", + self.document.remarks.prediction_target )); + writer.blank_line(); + + if self.document.models.h_theta.is_some() || self.document.models.k_h.is_some() { + writer.write("**Models:** "); + if let Some(model) = self.document.models.h_theta { + writer.write(format_args!("$h(\\theta)$ \u{2014} {model}")); + if self.document.models.k_h.is_some() { + writer.write("; "); + } + } + if let Some(model) = self.document.models.k_h { + writer.write(format_args!("$k(h)$ \u{2014} {model}")); + } + writer.blank_line(); + } + + ParameterTable { + title: "Inputs", + parameters: self.document.parameters, + } + .render(writer); + ParameterTable { + title: "Outputs", + parameters: match self.document.returns { + docs::Returns::Scalar(field) => std::slice::from_ref(field), + docs::Returns::Record { fields, .. } => fields, + }, + } + .render(writer); + for note in self.document.notes { + Admonition { + kind: "note", + body: note, + } + .render(writer); + } + for warning in self.document.warnings { + Admonition { + kind: "warning", + body: warning, + } + .render(writer); + } } - text.push('\n'); } -fn render_admonition(text: &mut String, kind: &str, body: &str) { - text.push_str(&format!("!!! {kind}\n\n")); - for line in body.lines() { - text.push_str(&format!(" {line}\n")); +struct ParameterTable<'a> { + title: &'a str, + parameters: &'a [Parameter], +} + +impl Render for ParameterTable<'_> { + fn render(&self, writer: &mut Writer) { + writer.line(format_args!("#### {}", self.title)); + writer.blank_line(); + writer.line("| Name | Unit | Domain | Description |"); + writer.line("| --- | --- | --- | --- |"); + for parameter in self.parameters { + writer.line(format_args!( + "| `{}` | {} | {} | {} |", + parameter.name, + escape_table(¶meter.unit), + parameter + .domain + .as_deref() + .map(escape_table) + .unwrap_or_else(|| "\u{2014}".into()), + escape_table(¶meter.description), + )); + } + writer.blank_line(); + } +} + +struct Admonition<'a> { + kind: &'a str, + body: &'a str, +} + +impl Render for Admonition<'_> { + fn render(&self, writer: &mut Writer) { + admonition(writer, self.kind, self.body, str::to_owned); } - text.push('\n'); } fn escape_table(value: &str) -> String { value.replace('\n', " ").replace('|', "\\|") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn renders_admonitions_with_writer_indentation() { + let mut writer = Writer::new(); + Admonition { + kind: "note", + body: "first\nsecond", + } + .render(&mut writer); + + assert_eq!( + writer.into_string(), + "!!! note\n\n first\n second\n\n" + ); + } + + #[test] + fn renders_generated_frontmatter_with_the_existing_spacing() { + let file = markdown_file("test.md", |writer| { + generated_frontmatter(writer, |writer| writer.line("title: Test")); + writer.line("# Test"); + }); + + assert_eq!( + file.contents, + "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: Test\n---\n\n# Test\n" + ); + } +} diff --git a/codegen/src/targets/python_documentation.rs b/codegen/src/targets/python_documentation.rs index d7557a6..6677793 100644 --- a/codegen/src/targets/python_documentation.rs +++ b/codegen/src/targets/python_documentation.rs @@ -1,67 +1,65 @@ -use std::path::PathBuf; - use crate::{ model::Entry, render::{Render, Writer}, }; -use super::{GeneratedFile, documentation::FRONTMATTER_HEADER, py::natural_sort_key}; +use super::{ + GeneratedFile, + documentation::{self as markdown}, + py::natural_sort_key, +}; pub(super) fn render(entries: &[Entry]) -> Vec { let mut entries = entries.iter().collect::>(); entries.sort_by_key(|entry| natural_sort_key(&entry.slug)); - let mut files = vec![file("index.md", index(&entries))]; + let mut files = vec![markdown::markdown_file("index.md", |writer| { + IndexPage { entries: &entries }.render(writer); + })]; for entry in entries { - files.push(page(&entry.slug)); + files.push(markdown::markdown_file( + format!("{}.md", entry.slug), + |writer| ModulePage { entry }.render(writer), + )); } files } -fn file(path: impl Into, contents: String) -> GeneratedFile { - GeneratedFile::new(path.into(), format!("{}\n", contents.trim_end())) +struct IndexPage<'a> { + entries: &'a [&'a Entry], } -fn index(entries: &[&Entry]) -> String { - let mut writer = Writer::new(); - writer.block( - "---", - false, - |writer| { - writer.write(FRONTMATTER_HEADER); +impl Render for IndexPage<'_> { + fn render(&self, writer: &mut Writer) { + markdown::generated_frontmatter(writer, |writer| { writer.line("title: Python API reference"); - }, - "---", - ); - writer.blank_line(); - writer.line("# Python API reference"); - writer.blank_line(); - writer.line("ptfkit's Python API is organized around public source modules."); - writer.blank_line(); - writer.line("## Modules"); - writer.blank_line(); - for entry in entries { - ModuleReference { entry }.render(&mut writer); + }); + writer.line("# Python API reference"); + writer.blank_line(); + writer.line("ptfkit's Python API is organized around public source modules."); + writer.blank_line(); + writer.line("## Modules"); + writer.blank_line(); + for entry in self.entries { + ModuleReference { entry }.render(writer); + } } - writer.into_string() } -fn page(slug: &str) -> GeneratedFile { - let module = format!("ptfkit.{slug}"); - let mut writer = Writer::new(); - writer.block( - "---", - false, - |writer| { - writer.write(FRONTMATTER_HEADER); +struct ModulePage<'a> { + entry: &'a Entry, +} + +impl Render for ModulePage<'_> { + fn render(&self, writer: &mut Writer) { + let slug = &self.entry.slug; + let module = format!("ptfkit.{slug}"); + markdown::generated_frontmatter(writer, |writer| { writer.line(format_args!("title: Python module {module}")); writer.line(format_args!("nav-title: {module}")); - }, - "---", - ); - writer.blank_line(); - writer.line(format_args!("::: {module}")); - file(format!("{slug}.md"), writer.into_string()) + }); + writer.line(format_args!("::: {module}")); + } } struct ModuleReference<'a> { @@ -120,7 +118,7 @@ mod tests { let page = contents(&files, &format!("{}.md", entry.slug)); assert!(page.starts_with(&format!( - "---\n{FRONTMATTER_HEADER}title: Python module {module}\nnav-title: {module}\n---\n" + "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: Python module {module}\nnav-title: {module}\n---\n" ))); assert!(page.contains(&format!("::: {module}"))); } From 4667bba87679b0455bc8ed70f951d463f2732caf Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Fri, 21 Aug 2026 10:30:17 +0300 Subject: [PATCH 12/14] refactor(codegen): organize modules by pipeline boundaries --- codegen/README.md | 30 +- codegen/src/{targets => }/compile.rs | 0 codegen/src/documentation.rs | 8 + codegen/src/main.rs | 2 + codegen/src/output/mod.rs | 169 +++++++++++ codegen/src/{targets => output}/write.rs | 21 +- .../{targets/c_expression.rs => render/c.rs} | 10 +- codegen/src/render/markdown.rs | 56 ++++ codegen/src/{render.rs => render/mod.rs} | 19 +- .../targets/{documentation.rs => catalog.rs} | 150 +++------- codegen/src/targets/mod.rs | 277 +++--------------- codegen/src/targets/native.rs | 158 ++++------ .../targets/{py/c.rs => python/extension.rs} | 104 ++++--- codegen/src/targets/{py => python}/mod.rs | 10 +- codegen/src/targets/{py => python}/stub.rs | 0 codegen/src/targets/{py => python}/syntax.rs | 4 - codegen/src/targets/{py => python}/test.rs | 28 +- codegen/src/targets/{py => python}/wrapper.rs | 21 +- .../{c_documentation.rs => reference/c.rs} | 108 +++---- .../cpp.rs} | 108 +++---- codegen/src/targets/reference/mod.rs | 5 + .../python.rs} | 21 +- codegen/src/targets/{rs.rs => rust.rs} | 12 +- .../ptfkit-native/cmake/ptfkitModules.cmake | 1 + 24 files changed, 593 insertions(+), 729 deletions(-) rename codegen/src/{targets => }/compile.rs (100%) create mode 100644 codegen/src/output/mod.rs rename codegen/src/{targets => output}/write.rs (95%) rename codegen/src/{targets/c_expression.rs => render/c.rs} (97%) create mode 100644 codegen/src/render/markdown.rs rename codegen/src/{render.rs => render/mod.rs} (77%) rename codegen/src/targets/{documentation.rs => catalog.rs} (54%) rename codegen/src/targets/{py/c.rs => python/extension.rs} (61%) rename codegen/src/targets/{py => python}/mod.rs (86%) rename codegen/src/targets/{py => python}/stub.rs (100%) rename codegen/src/targets/{py => python}/syntax.rs (93%) rename codegen/src/targets/{py => python}/test.rs (87%) rename codegen/src/targets/{py => python}/wrapper.rs (97%) rename codegen/src/targets/{c_documentation.rs => reference/c.rs} (81%) rename codegen/src/targets/{cpp_documentation.rs => reference/cpp.rs} (81%) create mode 100644 codegen/src/targets/reference/mod.rs rename codegen/src/targets/{python_documentation.rs => reference/python.rs} (90%) rename codegen/src/targets/{rs.rs => rust.rs} (98%) diff --git a/codegen/README.md b/codegen/README.md index 086d6ca..0dc31fd 100644 --- a/codegen/README.md +++ b/codegen/README.md @@ -8,23 +8,25 @@ contract; internal generator APIs are not. ## Pipeline 1. `specs` loads source specifications and `validate` checks their contracts. -2. `targets::compile` resolves formulas and golden cases into - `CompiledFunction` values. -3. Target renderers consume those values and return `GeneratedFile` artifacts. - `documentation` provides borrowed source/function facts to every renderer; - it deliberately contains no target markup. -4. `targets::run` assigns artifacts to declarative `Layout` values. -5. `targets::write` stages files, runs each layout's formatter, atomically - replaces targets, and removes obsolete marker-owned files. +2. `compile` resolves formulas and golden cases into `CompiledFunction` values. +3. `documentation` provides borrowed source/function facts without target + markup. `render` contains shared text, Markdown, and C-family expression + rendering support. +4. `targets::{catalog, reference, native, python, rust}` render concrete + generated products and return `GeneratedFile` artifacts. +5. `output` owns layouts, staging, formatter execution, cleanup, snapshots, + and atomic replacement. `check-generated` snapshots all marker-owned files, runs the same pipeline, and fails if the generated tree changes. ## Extension points -Add target-local syntax and escaping beside its renderer. Use -`render::Writer` only where indentation-aware text composition is natural; -Rust remains token-based with `proc_macro2` and `quote`. Add a `Layout` only -when a new output group needs a distinct root, cleanup root, marker, or -formatter. Do not introduce a cross-language syntax AST or let renderers own -filesystem, staging, formatting, or cleanup policy. +Add target-local syntax and escaping beside its renderer. Reusable C-family +expression precedence belongs in `render::c`; reusable Markdown file/block +composition belongs in `render::markdown`. Use `render::Writer` only where +indentation-aware text composition is natural; Rust remains token-based with +`proc_macro2` and `quote`. Add an `output::Layout` only when a new output group +needs a distinct root, cleanup root, marker, or formatter. Do not introduce a +cross-language syntax AST or let renderers own filesystem, staging, formatting, +or cleanup policy. diff --git a/codegen/src/targets/compile.rs b/codegen/src/compile.rs similarity index 100% rename from codegen/src/targets/compile.rs rename to codegen/src/compile.rs diff --git a/codegen/src/documentation.rs b/codegen/src/documentation.rs index 96616ae..e912483 100644 --- a/codegen/src/documentation.rs +++ b/codegen/src/documentation.rs @@ -93,6 +93,14 @@ pub(crate) fn for_function(function: &Function) -> FunctionDocument<'_> { } } +pub(crate) fn parameter_details(parameter: &Parameter) -> String { + format!("{} ({})", parameter.description, parameter.unit) +} + +pub(crate) fn parameter_documentation(parameter: &Parameter) -> String { + format!("{}: {}", parameter.name, parameter_details(parameter)) +} + #[cfg(test)] mod tests { use crate::model::{ diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 0570255..304aeef 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -5,9 +5,11 @@ use std::{path::Path, process::ExitCode}; use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; +mod compile; mod documentation; mod formula; mod model; +mod output; mod render; mod semantic; mod specs; diff --git a/codegen/src/output/mod.rs b/codegen/src/output/mod.rs new file mode 100644 index 0000000..546d2e2 --- /dev/null +++ b/codegen/src/output/mod.rs @@ -0,0 +1,169 @@ +//! Generated-artifact layout, formatting, and atomic filesystem commit. + +mod write; + +use std::path::PathBuf; + +pub(crate) use write::{assert_unchanged, commit, snapshot_generated}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Formatter { + None, + Rust, + Python, + C, + Cpp, +} + +pub(crate) struct Layout { + pub(super) output_directory: &'static str, + pub(super) cleanup_directory: &'static str, + pub(super) generated_header: &'static str, + pub(super) formatter: Formatter, +} + +macro_rules! layout { + ($name:ident, $output:literal, $cleanup:literal, $header:expr, $formatter:expr) => { + pub(crate) static $name: Layout = Layout { + output_directory: $output, + cleanup_directory: $cleanup, + generated_header: $header, + formatter: $formatter, + }; + }; +} + +const MARKDOWN_HEADER: &str = "\n\n"; +const RUST_HEADER: &str = "// @generated by ptfkit-codegen; DO NOT EDIT.\n"; +const C_HEADER: &str = "/* @generated by ptfkit-codegen; DO NOT EDIT. */\n"; +const PYTHON_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n"; + +layout!( + CATALOG, + "docs/src/ptf-catalog/sources", + "docs/src/ptf-catalog/sources", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + REFERENCE_C, + "docs/src/reference/c", + "docs/src/reference/c", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + REFERENCE_CPP, + "docs/src/reference/cpp", + "docs/src/reference/cpp", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + REFERENCE_PYTHON, + "docs/src/reference/python", + "docs/src/reference/python", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + RUST, + "targets/ptfkit-rs/src", + "targets/ptfkit-rs/src", + RUST_HEADER, + Formatter::Rust +); +layout!( + PYTHON_EXTENSION, + "targets/ptfkit-py", + "targets/ptfkit-py/src/ptfkit", + C_HEADER, + Formatter::C +); +layout!( + PYTHON_WRAPPER, + "targets/ptfkit-py/src", + "targets/ptfkit-py/src/ptfkit", + PYTHON_HEADER, + Formatter::Python +); +layout!( + PYTHON_TEST, + "targets/ptfkit-py", + "targets/ptfkit-py/tests", + PYTHON_HEADER, + Formatter::Python +); +layout!( + NATIVE_C, + "targets/ptfkit-native/include", + "targets/ptfkit-native/include", + C_HEADER, + Formatter::Cpp +); +layout!( + NATIVE_CPP_MODULE, + "targets/ptfkit-native/cpp", + "targets/ptfkit-native/cpp", + C_HEADER, + Formatter::Cpp +); +layout!( + NATIVE_CPP_CMAKE, + "targets/ptfkit-native/cmake", + "targets/ptfkit-native/cmake", + PYTHON_HEADER, + Formatter::None +); +layout!( + NATIVE_C_TEST, + "targets/ptfkit-native/tests/c", + "targets/ptfkit-native/tests/c", + C_HEADER, + Formatter::C +); +layout!( + NATIVE_CPP_TEST, + "targets/ptfkit-native/tests/cpp", + "targets/ptfkit-native/tests/cpp", + C_HEADER, + Formatter::Cpp +); + +pub(crate) const LAYOUTS: [&Layout; 13] = [ + &CATALOG, + &REFERENCE_C, + &REFERENCE_CPP, + &REFERENCE_PYTHON, + &RUST, + &PYTHON_EXTENSION, + &PYTHON_WRAPPER, + &PYTHON_TEST, + &NATIVE_C, + &NATIVE_CPP_MODULE, + &NATIVE_CPP_CMAKE, + &NATIVE_C_TEST, + &NATIVE_CPP_TEST, +]; + +pub(crate) struct Output { + pub(super) layout: &'static Layout, + pub(super) files: Vec, +} + +impl Output { + pub(crate) fn new(layout: &'static Layout, files: Vec) -> Self { + Self { layout, files } + } +} + +pub(crate) struct GeneratedFile { + pub(crate) path: PathBuf, + pub(crate) contents: String, +} + +impl GeneratedFile { + pub(crate) fn new(path: PathBuf, contents: String) -> Self { + Self { path, contents } + } +} diff --git a/codegen/src/targets/write.rs b/codegen/src/output/write.rs similarity index 95% rename from codegen/src/targets/write.rs rename to codegen/src/output/write.rs index 0f3f164..e862c7b 100644 --- a/codegen/src/targets/write.rs +++ b/codegen/src/output/write.rs @@ -19,9 +19,9 @@ struct StagedWrite { static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); -pub(super) struct GeneratedTree(BTreeMap>); +pub(crate) struct GeneratedTree(BTreeMap>); -pub(super) fn snapshot_generated(root: &Path) -> Result { +pub(crate) fn snapshot_generated(root: &Path) -> Result { let mut files = BTreeMap::new(); for layout in LAYOUTS { collect_generated( @@ -34,7 +34,7 @@ pub(super) fn snapshot_generated(root: &Path) -> Result { Ok(GeneratedTree(files)) } -pub(super) fn assert_unchanged(root: &Path, before: GeneratedTree) -> Result<()> { +pub(crate) fn assert_unchanged(root: &Path, before: GeneratedTree) -> Result<()> { let after = snapshot_generated(root)?; if let Some(report) = drift_report(&before.0, &after.0) { bail!("generated output drift after regeneration:\n{report}") @@ -112,7 +112,7 @@ fn append_paths(report: &mut String, label: &str, paths: &[&PathBuf]) { } } -pub(super) fn commit(root: &Path, outputs: &[Output]) -> Result<()> { +pub(crate) fn commit(root: &Path, outputs: &[Output]) -> Result<()> { let staged = stage(root, outputs)?; if let Err(error) = format(root, &staged) { remove_temporary(&staged); @@ -212,9 +212,9 @@ fn remove_obsolete(directory: &Path, expected: &BTreeSet, header: &str) fn is_generated(contents: &[u8], header: &str) -> bool { contents.starts_with(header.as_bytes()) - || (header == super::documentation::HEADER + || (header == super::MARKDOWN_HEADER && (contents.strip_prefix(b"---\n").is_some_and(|contents| { - contents.starts_with(super::documentation::FRONTMATTER_HEADER.as_bytes()) + contents.starts_with(crate::render::markdown::FRONTMATTER_HEADER.as_bytes()) }) || (contents.starts_with(b"---\n") && contents .windows(header.len()) @@ -311,7 +311,10 @@ mod tests { use std::{collections::BTreeMap, fs, path::Path}; use super::*; - use crate::targets::{GeneratedFile, Output, PYTHON_DOCUMENTATION, documentation::HEADER}; + use crate::{ + output::{GeneratedFile, Output, REFERENCE_PYTHON}, + render::markdown::HEADER, + }; fn temporary_root(label: &str) -> PathBuf { std::env::temp_dir().join(format!( @@ -339,7 +342,7 @@ mod tests { .expect("write C page"); let output = Output::new( - &PYTHON_DOCUMENTATION, + &REFERENCE_PYTHON, vec![GeneratedFile::new( "index.md".into(), format!("{HEADER}# Python\n"), @@ -363,7 +366,7 @@ mod tests { assert!(is_generated( format!( "---\n{}title: ptfkit.test\n---\n", - super::super::documentation::FRONTMATTER_HEADER + crate::render::markdown::FRONTMATTER_HEADER ) .as_bytes(), HEADER, diff --git a/codegen/src/targets/c_expression.rs b/codegen/src/render/c.rs similarity index 97% rename from codegen/src/targets/c_expression.rs rename to codegen/src/render/c.rs index 32200b7..1939645 100644 --- a/codegen/src/targets/c_expression.rs +++ b/codegen/src/render/c.rs @@ -3,12 +3,12 @@ use std::fmt; use crate::semantic::{BinaryOp, Expr, MathFunction, Reference, UnaryOp, Variable}; #[derive(Clone, Copy)] -pub(super) enum Dialect { +pub(crate) enum Dialect { C, Cpp, } -pub(super) fn expression<'a>( +pub(crate) fn expression<'a>( value: &'a Expr, inputs: &'a [String], variables: &'a [Variable], @@ -22,7 +22,7 @@ pub(super) fn expression<'a>( } } -pub(super) fn float_literal(lexeme: &str) -> String { +pub(crate) fn float_literal(lexeme: &str) -> String { if lexeme.contains(['.', 'e', 'E']) { lexeme.to_owned() } else { @@ -30,11 +30,11 @@ pub(super) fn float_literal(lexeme: &str) -> String { } } -pub(super) fn test_float_literal(value: f64) -> String { +pub(crate) fn test_float_literal(value: f64) -> String { float_literal(&value.to_string()) } -pub(super) fn requires_math(expression: &Expr) -> bool { +pub(crate) fn requires_math(expression: &Expr) -> bool { match expression { Expr::Number(_) | Expr::Reference(_) => false, Expr::Unary { operand, .. } => requires_math(operand), diff --git a/codegen/src/render/markdown.rs b/codegen/src/render/markdown.rs new file mode 100644 index 0000000..7ad8a48 --- /dev/null +++ b/codegen/src/render/markdown.rs @@ -0,0 +1,56 @@ +//! Shared Markdown file and block rendering primitives. + +use std::path::PathBuf; + +use super::Writer; +use crate::output::GeneratedFile; + +pub(crate) const HEADER: &str = "\n\n"; +pub(crate) const FRONTMATTER_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n\n"; + +pub(crate) fn markdown_file( + path: impl Into, + render: impl FnOnce(&mut Writer), +) -> GeneratedFile { + let mut writer = Writer::new(); + render(&mut writer); + GeneratedFile::new(path.into(), markdown_contents(writer)) +} + +pub(crate) fn markdown_contents(writer: Writer) -> String { + format!("{}\n", writer.into_string().trim_end()) +} + +pub(crate) fn frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { + writer.line("---"); + render(writer); + writer.write("---\n\n"); +} + +pub(crate) fn generated_frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { + frontmatter(writer, |writer| { + writer.write(FRONTMATTER_HEADER); + render(writer); + }); +} + +pub(crate) fn code_block(writer: &mut Writer, language: &str, render: impl FnOnce(&mut Writer)) { + writer.line(format_args!("```{language}")); + render(writer); + writer.write("```\n\n"); +} + +pub(crate) fn admonition( + writer: &mut Writer, + kind: &str, + body: &str, + render_line: impl Fn(&str) -> String, +) { + writer.write(format_args!("!!! {kind}\n\n")); + writer.indented(|writer| { + for line in body.lines() { + writer.line(render_line(line)); + } + }); + writer.blank_line(); +} diff --git a/codegen/src/render.rs b/codegen/src/render/mod.rs similarity index 77% rename from codegen/src/render.rs rename to codegen/src/render/mod.rs index 93c3527..55e6e3c 100644 --- a/codegen/src/render.rs +++ b/codegen/src/render/mod.rs @@ -1,5 +1,8 @@ use std::fmt::{self, Display, Write as _}; +pub(crate) mod c; +pub(crate) mod markdown; + /// Renders a structured value into a [`Writer`]. pub(crate) trait Render { fn render(&self, writer: &mut Writer); @@ -9,6 +12,7 @@ pub(crate) trait Render { pub(crate) struct Writer { contents: String, indentation: usize, + indent: &'static str, at_line_start: bool, } @@ -17,10 +21,22 @@ impl Writer { Self { contents: String::new(), indentation: 0, + indent: " ", at_line_start: true, } } + #[allow(dead_code)] + pub(crate) fn with_indent(mut self, indent: &'static str) -> Self { + self.indent = indent; + self + } + + /// Writes a text fragment, adding indentation only at the start of its lines. + /// + /// Keep large static templates as single fragments. Use [`Self::line`] and + /// [`Self::indented`] for dynamic or nested structure; templates must not + /// include their own leading indentation for a surrounding block. pub(crate) fn write(&mut self, value: impl Display) { self.write_fmt(format_args!("{value}")) .expect("writing to a String cannot fail"); @@ -38,6 +54,7 @@ impl Writer { self.write("\n"); } + /// Renders a nested block with one additional indentation level. pub(crate) fn indented(&mut self, render: impl FnOnce(&mut Self)) { self.indentation += 1; render(self); @@ -54,7 +71,7 @@ impl fmt::Write for Writer { for part in text.split_inclusive('\n') { if self.at_line_start && part != "\n" { for _ in 0..self.indentation { - self.contents.push_str(" "); + self.contents.push_str(self.indent); } } self.contents.push_str(part); diff --git a/codegen/src/targets/documentation.rs b/codegen/src/targets/catalog.rs similarity index 54% rename from codegen/src/targets/documentation.rs rename to codegen/src/targets/catalog.rs index 9b0024b..5bf3284 100644 --- a/codegen/src/targets/documentation.rs +++ b/codegen/src/targets/catalog.rs @@ -1,81 +1,21 @@ -use std::path::PathBuf; - use crate::{ documentation::{self as docs, FunctionDocument}, model::{Entry, Parameter}, - render::{Render, Writer}, + output::GeneratedFile, + render::{Render, Writer, markdown}, }; -pub(super) const HEADER: &str = "\n\n"; -pub(super) const FRONTMATTER_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n\n"; - -pub(super) fn parameter_details(parameter: &Parameter) -> String { - format!("{} ({})", parameter.description, parameter.unit) -} - -pub(super) fn parameter_documentation(parameter: &Parameter) -> String { - format!("{}: {}", parameter.name, parameter_details(parameter)) -} - -pub(super) fn markdown_file( - path: impl Into, - render: impl FnOnce(&mut Writer), -) -> super::GeneratedFile { - let mut writer = Writer::new(); - render(&mut writer); - super::GeneratedFile::new(path.into(), markdown_contents(writer)) -} - -pub(super) fn markdown_contents(writer: Writer) -> String { - let contents = writer.into_string(); - format!("{}\n", contents.trim_end()) -} - -pub(super) fn frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { - writer.line("---"); - render(writer); - writer.line("---"); - writer.blank_line(); -} - -pub(super) fn generated_frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { - frontmatter(writer, |writer| { - writer.write(FRONTMATTER_HEADER); - render(writer); - }); -} - -pub(super) fn code_block(writer: &mut Writer, language: &str, render: impl FnOnce(&mut Writer)) { - writer.line(format_args!("```{language}")); - render(writer); - writer.line("```"); - writer.blank_line(); -} - -pub(super) fn admonition( - writer: &mut Writer, - kind: &str, - body: &str, - render_line: impl Fn(&str) -> String, -) { - writer.line(format_args!("!!! {kind}")); - writer.blank_line(); - writer.indented(|writer| { - for line in body.lines() { - writer.line(render_line(line)); - } - }); - writer.blank_line(); -} - -pub(super) fn render(entries: &[Entry]) -> Vec { - let mut files = vec![markdown_file("index.md", |writer| { +pub(super) fn render(entries: &[Entry]) -> Vec { + let mut files = vec![markdown::markdown_file("index.md", |writer| { IndexPage { entries }.render(writer); })]; for entry in entries { - files.push(markdown_file(format!("{}.md", entry.slug), |writer| { - SourcePage { entry }.render(writer); - })); + files.push(markdown::markdown_file( + format!("{}.md", entry.slug), + |writer| { + SourcePage { entry }.render(writer); + }, + )); } files } @@ -86,15 +26,15 @@ struct IndexPage<'a> { impl Render for IndexPage<'_> { fn render(&self, writer: &mut Writer) { - generated_frontmatter(writer, |writer| writer.line("title: PTF Sources")); - writer.line("# PTF Sources"); - writer.blank_line(); - writer.line( - "Each page describes the source, scope, inputs, outputs, status, and limitations of the functions defined by one specification.", + markdown::generated_frontmatter(writer, |writer| writer.line("title: PTF Sources")); + writer.write( + "# PTF Sources\n\ +\n\ +Each page describes the source, scope, inputs, outputs, status, and limitations of the functions defined by one specification.\n\ +\n\ +| Source | Territory | Functions |\n\ +| --- | --- | ---: |\n", ); - writer.blank_line(); - writer.line("| Source | Territory | Functions |"); - writer.line("| --- | --- | ---: |"); for entry in self.entries { let source = docs::for_source(&entry.spec.source, &entry.spec.scope); let territory = source.territory.unwrap_or("\u{2014}"); @@ -117,34 +57,27 @@ impl Render for SourcePage<'_> { fn render(&self, writer: &mut Writer) { let spec = &self.entry.spec; let source = docs::for_source(&spec.source, &spec.scope); - generated_frontmatter(writer, |writer| { + markdown::generated_frontmatter(writer, |writer| { writer.line(format_args!("title: PTF source {}", self.entry.slug)); writer.line(format_args!("nav-title: {}", self.entry.slug)); }); - writer.line(format_args!("# {}", source.summary)); - writer.blank_line(); - writer.line("## Source"); - writer.blank_line(); - writer.line(source.reference.citation); - writer.blank_line(); + writer.write(format_args!( + "# {}\n\n## Source\n\n{}\n\n", + source.summary, source.reference.citation + )); if let Some(doi) = source.reference.doi { - writer.line(format_args!("[DOI: {}]({})", doi.identifier, doi.url)); - writer.blank_line(); + writer.write(format_args!("[DOI: {}]({})\n\n", doi.identifier, doi.url)); } if source.territory.is_some() || source.dataset.is_some() { - writer.line("## Scope"); - writer.blank_line(); + writer.write("## Scope\n\n"); if let Some(territory) = source.territory { - writer.line(format_args!("**Territory:** {territory}")); - writer.blank_line(); + writer.write(format_args!("**Territory:** {territory}\n\n")); } if let Some(dataset) = source.dataset { - writer.line(format_args!("**Dataset:** {dataset}")); - writer.blank_line(); + writer.write(format_args!("**Dataset:** {dataset}\n\n")); } } - writer.line("## Functions"); - writer.blank_line(); + writer.write("## Functions\n\n"); for function in &spec.functions { FunctionSection { document: docs::for_function(function), @@ -164,17 +97,10 @@ struct FunctionSection<'a> { impl Render for FunctionSection<'_> { fn render(&self, writer: &mut Writer) { - writer.line(format_args!("### `{}`", self.name)); - writer.blank_line(); - writer.line(self.document.summary); - writer.blank_line(); - writer.line(format_args!("**Status:** `{}`", self.status)); - writer.blank_line(); - writer.line(format_args!( - "**Prediction target:** {}", - self.document.remarks.prediction_target + writer.write(format_args!( + "### `{}`\n\n{}\n\n**Status:** `{}`\n\n**Prediction target:** {}\n\n", + self.name, self.document.summary, self.status, self.document.remarks.prediction_target, )); - writer.blank_line(); if self.document.models.h_theta.is_some() || self.document.models.k_h.is_some() { writer.write("**Models:** "); @@ -227,10 +153,10 @@ struct ParameterTable<'a> { impl Render for ParameterTable<'_> { fn render(&self, writer: &mut Writer) { - writer.line(format_args!("#### {}", self.title)); - writer.blank_line(); - writer.line("| Name | Unit | Domain | Description |"); - writer.line("| --- | --- | --- | --- |"); + writer.write(format_args!( + "#### {}\n\n| Name | Unit | Domain | Description |\n| --- | --- | --- | --- |\n", + self.title + )); for parameter in self.parameters { writer.line(format_args!( "| `{}` | {} | {} | {} |", @@ -255,7 +181,7 @@ struct Admonition<'a> { impl Render for Admonition<'_> { fn render(&self, writer: &mut Writer) { - admonition(writer, self.kind, self.body, str::to_owned); + markdown::admonition(writer, self.kind, self.body, str::to_owned); } } @@ -284,8 +210,8 @@ mod tests { #[test] fn renders_generated_frontmatter_with_the_existing_spacing() { - let file = markdown_file("test.md", |writer| { - generated_frontmatter(writer, |writer| writer.line("title: Test")); + let file = markdown::markdown_file("test.md", |writer| { + markdown::generated_frontmatter(writer, |writer| writer.line("title: Test")); writer.line("# Test"); }); diff --git a/codegen/src/targets/mod.rs b/codegen/src/targets/mod.rs index eaa5096..b0d1afc 100644 --- a/codegen/src/targets/mod.rs +++ b/codegen/src/targets/mod.rs @@ -1,22 +1,20 @@ -mod c_documentation; -mod c_expression; -mod compile; -mod cpp_documentation; -mod documentation; +//! Render concrete generated products from compiled specifications. + +mod catalog; mod native; -mod py; -mod python_documentation; -mod rs; -mod write; +mod python; +mod reference; +mod rust; -use std::{ - collections::BTreeMap, - path::{Path, PathBuf}, -}; +use std::{collections::BTreeMap, path::Path}; use anyhow::Result; -use crate::model::{CompiledFunction, Entry}; +use crate::{ + compile, + model::{CompiledFunction, Entry}, + output::{self, Output}, +}; pub(super) fn group_by_source( functions: &[CompiledFunction], @@ -31,244 +29,39 @@ pub(super) fn group_by_source( sources } -#[derive(Clone, Copy, PartialEq, Eq)] -pub(super) enum Formatter { - None, - Rust, - Python, - C, - Cpp, -} - -pub(super) struct Layout { - pub(super) output_directory: &'static str, - pub(super) cleanup_directory: &'static str, - pub(super) generated_header: &'static str, - pub(super) formatter: Formatter, -} - -macro_rules! layout { - ( - $name:ident, - $output_directory:literal, - $cleanup_directory:literal, - $generated_header:path, - $formatter:path, - ) => { - pub(super) static $name: Layout = Layout { - output_directory: $output_directory, - cleanup_directory: $cleanup_directory, - generated_header: $generated_header, - formatter: $formatter, - }; - }; -} - -layout!( - DOCUMENTATION, - "docs/src/ptf-catalog/sources", - "docs/src/ptf-catalog/sources", - documentation::HEADER, - Formatter::None, -); -layout!( - C_DOCUMENTATION, - "docs/src/reference/c", - "docs/src/reference/c", - documentation::HEADER, - Formatter::None, -); -layout!( - CPP_DOCUMENTATION, - "docs/src/reference/cpp", - "docs/src/reference/cpp", - documentation::HEADER, - Formatter::None, -); -layout!( - PYTHON_DOCUMENTATION, - "docs/src/reference/python", - "docs/src/reference/python", - documentation::HEADER, - Formatter::None, -); -layout!( - RUST, - "targets/ptfkit-rs/src", - "targets/ptfkit-rs/src", - rs::HEADER, - Formatter::Rust, -); -layout!( - PYTHON_EXTENSION, - "targets/ptfkit-py", - "targets/ptfkit-py/src/ptfkit", - py::C_HEADER, - Formatter::C, -); -layout!( - PYTHON_WRAPPER, - "targets/ptfkit-py/src", - "targets/ptfkit-py/src/ptfkit", - py::WRAPPER_HEADER, - Formatter::Python, -); -layout!( - PYTHON_TEST, - "targets/ptfkit-py", - "targets/ptfkit-py/tests", - py::WRAPPER_HEADER, - Formatter::Python, -); -layout!( - NATIVE_C, - "targets/ptfkit-native/include", - "targets/ptfkit-native/include", - native::HEADER, - Formatter::Cpp, -); -layout!( - NATIVE_CPP_MODULE, - "targets/ptfkit-native/cpp", - "targets/ptfkit-native/cpp", - native::HEADER, - Formatter::Cpp, -); -layout!( - NATIVE_CPP_CMAKE, - "targets/ptfkit-native/cmake", - "targets/ptfkit-native/cmake", - native::CMAKE_HEADER, - Formatter::None, -); -layout!( - NATIVE_C_TEST, - "targets/ptfkit-native/tests/c", - "targets/ptfkit-native/tests/c", - native::HEADER, - Formatter::C, -); -layout!( - NATIVE_CPP_TEST, - "targets/ptfkit-native/tests/cpp", - "targets/ptfkit-native/tests/cpp", - native::HEADER, - Formatter::Cpp, -); - -pub(super) const LAYOUTS: [&Layout; 13] = [ - &DOCUMENTATION, - &C_DOCUMENTATION, - &CPP_DOCUMENTATION, - &PYTHON_DOCUMENTATION, - &RUST, - &PYTHON_EXTENSION, - &PYTHON_WRAPPER, - &PYTHON_TEST, - &NATIVE_C, - &NATIVE_CPP_MODULE, - &NATIVE_CPP_CMAKE, - &NATIVE_C_TEST, - &NATIVE_CPP_TEST, -]; - -pub(super) struct Output { - pub(super) layout: &'static Layout, - pub(super) files: Vec, -} - -impl Output { - fn new(layout: &'static Layout, files: Vec) -> Self { - Self { layout, files } - } -} - -pub(super) struct GeneratedFile { - path: PathBuf, - contents: String, -} - -impl GeneratedFile { - fn new(path: PathBuf, contents: String) -> Self { - Self { path, contents } - } -} - pub(crate) fn run(root: &Path, entries: Vec) -> Result<()> { - let documentation = documentation::render(&entries); - let python_documentation = python_documentation::render(&entries); + let catalog = catalog::render(&entries); + let reference_python = reference::python::render(&entries); let compiled = compile::functions(entries)?; - let rust = rs::render(&compiled)?; - let py = py::render(&compiled)?; + let reference_c = reference::c::render(&compiled)?; + let reference_cpp = reference::cpp::render(&compiled)?; + let rust = rust::render(&compiled)?; + let python = python::render(&compiled)?; let native = native::render(&compiled)?; - let c_documentation = c_documentation::render(&compiled)?; - let cpp_documentation = cpp_documentation::render(&compiled)?; - write::commit( + + output::commit( root, &[ - Output::new(&DOCUMENTATION, documentation), - Output::new(&C_DOCUMENTATION, c_documentation), - Output::new(&CPP_DOCUMENTATION, cpp_documentation), - Output::new(&PYTHON_DOCUMENTATION, python_documentation), - Output::new(&RUST, rust), - Output::new(&PYTHON_EXTENSION, py.c_sources), - Output::new(&PYTHON_WRAPPER, py.wrappers), - Output::new(&PYTHON_TEST, py.tests), - Output::new(&NATIVE_C, native.c_headers), - Output::new(&NATIVE_CPP_MODULE, native.cpp_modules), - Output::new(&NATIVE_CPP_CMAKE, native.cpp_cmake), - Output::new(&NATIVE_C_TEST, native.c_tests), - Output::new(&NATIVE_CPP_TEST, native.cpp_tests), + Output::new(&output::CATALOG, catalog), + Output::new(&output::REFERENCE_C, reference_c), + Output::new(&output::REFERENCE_CPP, reference_cpp), + Output::new(&output::REFERENCE_PYTHON, reference_python), + Output::new(&output::RUST, rust), + Output::new(&output::PYTHON_EXTENSION, python.extension), + Output::new(&output::PYTHON_WRAPPER, python.wrappers), + Output::new(&output::PYTHON_TEST, python.tests), + Output::new(&output::NATIVE_C, native.c_headers), + Output::new(&output::NATIVE_CPP_MODULE, native.cpp_modules), + Output::new(&output::NATIVE_CPP_CMAKE, native.cpp_cmake), + Output::new(&output::NATIVE_C_TEST, native.c_tests), + Output::new(&output::NATIVE_CPP_TEST, native.cpp_tests), ], ) } /// Regenerate every target and fail when that changes a codegen-owned file. pub(crate) fn check_generated(root: &Path, entries: Vec) -> Result<()> { - let before = write::snapshot_generated(root)?; + let before = output::snapshot_generated(root)?; run(root, entries)?; - write::assert_unchanged(root, before) -} - -#[cfg(test)] -mod tests { - use std::path::Path; - - use super::{ - C_DOCUMENTATION, CPP_DOCUMENTATION, DOCUMENTATION, PYTHON_DOCUMENTATION, PYTHON_EXTENSION, - PYTHON_WRAPPER, - }; - - #[test] - fn documentation_targets_use_structured_mkdocs_roots() { - let root = Path::new("repository"); - let relative = Path::new("index.md"); - - for (layout, directory) in [ - (&DOCUMENTATION, "docs/src/ptf-catalog/sources"), - (&C_DOCUMENTATION, "docs/src/reference/c"), - (&CPP_DOCUMENTATION, "docs/src/reference/cpp"), - (&PYTHON_DOCUMENTATION, "docs/src/reference/python"), - ] { - assert_eq!( - root.join(layout.output_directory).join(relative), - root.join(directory).join(relative) - ); - assert_eq!(root.join(layout.cleanup_directory), root.join(directory)); - } - } - - #[test] - fn python_layouts_keep_extension_cleanup_scoped_to_its_package() { - assert_eq!(PYTHON_EXTENSION.output_directory, "targets/ptfkit-py"); - assert_eq!( - PYTHON_EXTENSION.cleanup_directory, - "targets/ptfkit-py/src/ptfkit" - ); - assert_eq!(PYTHON_WRAPPER.output_directory, "targets/ptfkit-py/src"); - assert_eq!( - PYTHON_WRAPPER.cleanup_directory, - "targets/ptfkit-py/src/ptfkit" - ); - } + output::assert_unchanged(root, before) } diff --git a/codegen/src/targets/native.rs b/codegen/src/targets/native.rs index 54ff69f..6cb33da 100644 --- a/codegen/src/targets/native.rs +++ b/codegen/src/targets/native.rs @@ -9,20 +9,21 @@ use crate::{ render::{Render, Writer}, }; -use super::{ - c_expression::{self, Dialect}, - documentation, group_by_source, +use crate::{ + output::GeneratedFile, + render::c::{self, Dialect}, + targets::group_by_source, }; pub(super) const HEADER: &str = "/* @generated by ptfkit-codegen; DO NOT EDIT. */\n"; pub(super) const CMAKE_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n"; pub(super) struct OutputFiles { - pub(super) c_headers: Vec, - pub(super) cpp_modules: Vec, - pub(super) cpp_cmake: Vec, - pub(super) c_tests: Vec, - pub(super) cpp_tests: Vec, + pub(super) c_headers: Vec, + pub(super) cpp_modules: Vec, + pub(super) cpp_cmake: Vec, + pub(super) c_tests: Vec, + pub(super) cpp_tests: Vec, } pub(super) fn render(functions: &[CompiledFunction]) -> Result { @@ -30,44 +31,34 @@ pub(super) fn render(functions: &[CompiledFunction]) -> Result { let mut cpp_modules = Vec::new(); let mut c_tests = Vec::new(); let mut cpp_tests = Vec::new(); - let mut c_includes = Writer::new(); + let mut umbrella = Writer::new(); + umbrella.write(format_args!( + "{HEADER}\n\n#ifndef PTFKIT_PTFKIT_H\n#define PTFKIT_PTFKIT_H\n\n" + )); + + let mut root_module = Writer::new(); + root_module.write(format_args!("{HEADER}\n\nexport module ptfkit;\n\n")); + let mut module_paths = vec!["cpp/ptfkit.cppm".to_owned()]; for (slug, functions) in group_by_source(functions) { - c_includes.line(format_args!("#include ")); + umbrella.line(format_args!("#include ")); c_headers.push(file( format!("ptfkit/{slug}.h"), c_header(slug, &functions)?, )); module_paths.push(format!("cpp/{slug}.cppm")); + root_module.line(format_args!("export import ptfkit.{slug};")); cpp_modules.push(file(format!("{slug}.cppm"), cpp_module(slug, &functions)?)); c_tests.push(file(format!("{slug}.c"), c_test(slug, &functions)?)); cpp_tests.push(file(format!("{slug}.cpp"), cpp_test(slug, &functions)?)); } - let mut umbrella = Writer::new(); - umbrella.write(HEADER); - umbrella.blank_line(); - umbrella.line("#ifndef PTFKIT_PTFKIT_H"); - umbrella.line("#define PTFKIT_PTFKIT_H"); - umbrella.blank_line(); - umbrella.write(c_includes.into_string()); - umbrella.blank_line(); - umbrella.line("#endif"); + umbrella.write("\n\n#endif\n"); c_headers.push(file("ptfkit/ptfkit.h", umbrella.into_string())); - let mut root_module = Writer::new(); - root_module.write(HEADER); - root_module.blank_line(); - root_module.line("export module ptfkit;"); - root_module.blank_line(); - for path in module_paths.iter().skip(1) { - let slug = path.trim_start_matches("cpp/").trim_end_matches(".cppm"); - root_module.line(format_args!("export import ptfkit.{slug};")); - } cpp_modules.push(file("ptfkit.cppm", root_module.into_string())); let mut cmake = Writer::new(); - cmake.write(CMAKE_HEADER); - cmake.line("set(PTFKIT_CPP_MODULES"); + cmake.write(format_args!("{CMAKE_HEADER}\nset(PTFKIT_CPP_MODULES\n")); cmake.indented(|writer| { for path in &module_paths { writer.line(format_args!("\"${{CMAKE_CURRENT_LIST_DIR}}/../{path}\"")); @@ -83,11 +74,11 @@ pub(super) fn render(functions: &[CompiledFunction]) -> Result { }) } -fn file(path: impl Into, contents: String) -> super::GeneratedFile { - super::GeneratedFile::new(path.into(), contents) +fn file(path: impl Into, contents: String) -> GeneratedFile { + GeneratedFile::new(path.into(), contents) } -pub(super) fn c_result_name(schema: &str) -> String { +pub(crate) fn c_result_name(schema: &str) -> String { schema .remove_boundaries(&[Boundary::LowerDigit]) .to_case(Case::Snake) @@ -96,14 +87,11 @@ pub(super) fn c_result_name(schema: &str) -> String { fn c_header(slug: &str, functions: &[&CompiledFunction]) -> Result { let guard = format!("PTFKIT_{}_H", slug.to_ascii_uppercase()); let mut writer = Writer::new(); - writer.write(HEADER); - writer.blank_line(); - writer.line(format_args!("#ifndef {guard}")); - writer.line(format_args!("#define {guard}")); - writer.blank_line(); + writer.write(format_args!( + "{HEADER}\n\n#ifndef {guard}\n#define {guard}\n\n" + )); if requires_math(functions) { - writer.line("#include "); - writer.blank_line(); + writer.write("#include \n\n"); } let first = functions .first() @@ -132,22 +120,17 @@ fn c_header(slug: &str, functions: &[&CompiledFunction]) -> Result { writer.blank_line(); NativeFunction::c(function)?.render(&mut writer); } - writer.blank_line(); - writer.line("#endif"); + writer.write("\n\n#endif\n"); Ok(writer.into_string()) } fn cpp_module(slug: &str, functions: &[&CompiledFunction]) -> Result { let mut writer = Writer::new(); - writer.write(HEADER); - writer.blank_line(); + writer.write(format_args!("{HEADER}\n\n")); if requires_math(functions) { - writer.line("module;"); - writer.line("#include "); - writer.blank_line(); + writer.write("module;\n#include \n\n"); } - writer.line(format_args!("export module ptfkit.{slug};")); - writer.blank_line(); + writer.write(format_args!("export module ptfkit.{slug};\n\n")); let first = functions .first() .expect("generated source contains at least one function"); @@ -177,8 +160,7 @@ fn cpp_module(slug: &str, functions: &[&CompiledFunction]) -> Result { function.render(writer); } }); - writer.blank_line(); - writer.line(format_args!("}} // namespace ptfkit::{slug}")); + writer.write(format_args!("\n\n}} // namespace ptfkit::{slug}\n")); Ok(writer.into_string()) } @@ -264,7 +246,7 @@ impl Render for NativeFunction<'_> { .take(self.function.ir.variables.len() - usize::from(self.terminal)) { writer.write(format_args!("const double {} = ", variable.name)); - writer.write(c_expression::expression( + writer.write(c::expression( &variable.expression, &self.function.core.inputs, &self.function.ir.variables, @@ -294,7 +276,7 @@ impl NativeFunction<'_> { match &self.function.core.output { Output::Scalar if self.terminal => { writer.write("return "); - writer.write(c_expression::expression( + writer.write(c::expression( &self .function .ir @@ -310,19 +292,15 @@ impl NativeFunction<'_> { } Output::Scalar => writer.line(format_args!("return {output_name};")), Output::Struct(fields) if matches!(self.dialect, NativeDialect::C) => { - writer.line("#ifdef __cplusplus"); - writer.write(format_args!("return {}{{", self.result)); + writer.write(format_args!("#ifdef __cplusplus\nreturn {}{{", self.result)); render_values(writer, fields); - writer.line("};"); - writer.line("#else"); - writer.line(format_args!("return ({}) {{", self.result)); + writer.write(format_args!("}};\n#else\nreturn ({}) {{\n", self.result)); writer.indented(|writer| { for field in fields { writer.line(format_args!(".{field} = {field},")); } }); - writer.line("};"); - writer.line("#endif"); + writer.write("};\n#endif\n"); } Output::Struct(fields) => { writer.write(format_args!("return {}{{", self.result)); @@ -412,7 +390,7 @@ fn function_comment_from_document(document: FunctionDocument<'_>) -> Comment { format!( "@param {} {}", parameter.name, - documentation::parameter_details(parameter) + docs::parameter_details(parameter) ) })); @@ -421,17 +399,14 @@ fn function_comment_from_document(document: FunctionDocument<'_>) -> Comment { docs::Returns::Record { fields, .. } => fields, }; if outputs.len() == 1 { - lines.push(format!( - "@return {}", - documentation::parameter_details(&outputs[0]) - )); + lines.push(format!("@return {}", docs::parameter_details(&outputs[0]))); } else { lines.push("@return A result with the following fields:".to_owned()); lines.extend(outputs.iter().map(|parameter| { format!( "- `{}` — {}", parameter.name, - documentation::parameter_details(parameter) + docs::parameter_details(parameter) ) })); } @@ -460,7 +435,7 @@ fn function_comment_from_document(document: FunctionDocument<'_>) -> Comment { fn field_comment(parameter: &Parameter) -> Comment { Comment(vec![format!( "@brief {}", - documentation::parameter_details(parameter) + docs::parameter_details(parameter) )]) } @@ -507,7 +482,7 @@ fn requires_math(functions: &[&CompiledFunction]) -> bool { .ir .variables .iter() - .any(|variable| c_expression::requires_math(&variable.expression)) + .any(|variable| c::requires_math(&variable.expression)) }) } @@ -520,11 +495,9 @@ fn cpp_test(slug: &str, functions: &[&CompiledFunction]) -> Result { fn c_compatibility_test(slug: &str, functions: &[&CompiledFunction]) -> Result { let mut writer = Writer::new(); - writer.write(HEADER); - writer.blank_line(); - writer.line(format_args!("#include ")); - writer.line("#include \"close_enough.h\""); - writer.blank_line(); + writer.write(format_args!( + "{HEADER}\n\n#include \n#include \"close_enough.h\"\n\n" + )); writer.line("int main() {"); writer.indented(|writer| { for function in functions { @@ -553,19 +526,19 @@ fn c_compatibility_test(slug: &str, functions: &[&CompiledFunction]) -> Result Result { .iter() .any(|function| matches!(function.core.output, Output::Struct(_))); let mut writer = Writer::new(); - writer.write(HEADER); - writer.blank_line(); - writer.line("#ifdef IMPORT_UMBRELLA"); - writer.line("import ptfkit;"); - writer.line("#else"); - writer.line(format_args!("import ptfkit.{slug};")); - writer.line("#endif"); - writer.blank_line(); - writer.line("#include \"close_enough.h\""); + writer.write(format_args!("{HEADER}\n\n#ifdef IMPORT_UMBRELLA\nimport ptfkit;\n#else\nimport ptfkit.{slug};\n#endif\n\n#include \"close_enough.h\"")); if type_traits { - writer.line("#include "); + writer.write("\n#include \n\n"); + } else { + writer.blank_line(); } - writer.blank_line(); writer.line("int main() {"); writer.indented(|writer| { for function in functions { @@ -612,19 +578,19 @@ fn module_test(slug: &str, functions: &[&CompiledFunction]) -> Result { writer.write(format_args!("result.{}", field.name)); } writer.write(", "); - writer.write(c_expression::test_float_literal(*expected)); + writer.write(c::test_float_literal(*expected)); writer.write(", "); - writer.write(c_expression::test_float_literal(case.atol)); + writer.write(c::test_float_literal(case.atol)); writer.write(", "); - writer.write(c_expression::test_float_literal(case.rtol)); + writer.write(c::test_float_literal(case.rtol)); writer.line(");"); } }); writer.line("}"); } } + writer.line("return 0;"); }); - writer.line(" return 0;"); writer.line("}"); Ok(writer.into_string()) } @@ -634,7 +600,7 @@ fn render_literals(writer: &mut Writer, values: &[f64]) { if index > 0 { writer.write(", "); } - writer.write(c_expression::test_float_literal(*value)); + writer.write(c::test_float_literal(*value)); } } @@ -664,9 +630,9 @@ mod tests { args: vec![value], }; - assert!(!c_expression::requires_math(&arithmetic)); - assert!(c_expression::requires_math(&power)); - assert!(c_expression::requires_math(&logarithm)); + assert!(!c::requires_math(&arithmetic)); + assert!(c::requires_math(&power)); + assert!(c::requires_math(&logarithm)); } #[test] diff --git a/codegen/src/targets/py/c.rs b/codegen/src/targets/python/extension.rs similarity index 61% rename from codegen/src/targets/py/c.rs rename to codegen/src/targets/python/extension.rs index 82ac41c..1599f15 100644 --- a/codegen/src/targets/py/c.rs +++ b/codegen/src/targets/python/extension.rs @@ -1,44 +1,38 @@ use anyhow::Result; -use crate::model::{CompiledFunction, Output}; -use crate::render::Writer; - -use super::{ - super::{ - GeneratedFile, - c_expression::{self, Dialect}, - group_by_source, +use crate::{ + model::{CompiledFunction, Output}, + output::GeneratedFile, + render::{ + Writer, + c::{self, Dialect}, }, - C_HEADER, + targets::group_by_source, }; +use super::C_HEADER; + pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { - let mut includes = Writer::new(); - let mut registers = Vec::new(); + let sources = group_by_source(functions); let mut writes = Vec::new(); - for (slug, functions) in group_by_source(functions) { + for (slug, functions) in &sources { let register = format!("ptfkit_register_{slug}"); - registers.push(register.clone()); - let mut definitions = Writer::new(); - let mut calls = Writer::new(); - for function in functions { - definitions.write(ufunc(function)?); - calls.line(format_args!( - "if (ptfkit_add_ufunc(module, \"{name}\", {name}_functions, {name}_types, {nin}, {nout}) < 0) return -1;", - name = function.core.name, - nin = function.core.inputs.len(), - nout = output_count(&function.core.output), - )); - } - includes.line(format_args!("#include \"{slug}.c\"")); let mut source = Writer::new(); source.write(C_HEADER); - source.line("#include \"ufunc.h\""); - source.blank_line(); - source.write(definitions.into_string()); + source.write("#include \"ufunc.h\"\n\n"); + for function in functions { + source.write(ufunc(function)?); + } source.line(format_args!("int {register}(PyObject *module) {{")); source.indented(|writer| { - writer.write(calls.into_string()); + for function in functions { + writer.line(format_args!( + "if (ptfkit_add_ufunc(module, \"{name}\", {name}_functions, {name}_types, {nin}, {nout}) < 0) return -1;", + name = function.core.name, + nin = function.core.inputs.len(), + nout = output_count(&function.core.output), + )); + } writer.line("return 0;"); }); source.line("}"); @@ -47,31 +41,33 @@ pub(crate) fn render(functions: &[CompiledFunction]) -> Result"); - entry.line("#include "); - entry.line("#include "); - entry.blank_line(); - entry.write(includes.into_string()); - entry.blank_line(); - entry.line("static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, \"_ptfkit\", NULL, -1, NULL };"); + entry.write( + r#"#define PY_SSIZE_T_CLEAN +#define PY_ARRAY_UNIQUE_SYMBOL PTFKIT_ARRAY_API +#include +#include +#include "#, + ); entry.blank_line(); - entry.line("PyMODINIT_FUNC PyInit__ptfkit(void) {"); + for slug in sources.keys() { + entry.line(format_args!("#include \"{slug}.c\"")); + } + entry.write("\n\nstatic struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, \"_ptfkit\", NULL, -1, NULL };\n\nPyMODINIT_FUNC PyInit__ptfkit(void) {\n"); entry.indented(|writer| { - writer.line("PyObject *module = PyModule_Create(&module_def);"); - writer.line("if (module == NULL) return NULL;"); - writer.line("import_array();"); - writer.line("import_ufunc();"); - writer.write(calls.into_string()); + writer.write( + r#"PyObject *module = PyModule_Create(&module_def); +if (module == NULL) return NULL; +import_array(); +import_ufunc(); +"#, + ); + for slug in sources.keys() { + writer.line(format_args!( + "if (ptfkit_register_{slug}(module) < 0) {{ Py_DECREF(module); return NULL; }}" + )); + } writer.line("return module;"); }); entry.line("}"); @@ -111,7 +107,7 @@ fn ufunc(function: &CompiledFunction) -> Result { } for variable in &function.ir.variables { writer.write(format_args!("const double {} = ", variable.name)); - writer.write(c_expression::expression( + writer.write(c::expression( &variable.expression, inputs, &function.ir.variables, @@ -133,11 +129,9 @@ fn ufunc(function: &CompiledFunction) -> Result { writer.line("}"); }); writer.line("}"); - writer.line(format_args!( - "static PyUFuncGenericFunction {name}_functions[] = {{ {name}_loop }};" + writer.write(format_args!( + "static PyUFuncGenericFunction {name}_functions[] = {{ {name}_loop }};\nstatic char {name}_types[] = {{ {types} }};\n\n" )); - writer.line(format_args!("static char {name}_types[] = {{ {types} }};")); - writer.blank_line(); Ok(writer.into_string()) } diff --git a/codegen/src/targets/py/mod.rs b/codegen/src/targets/python/mod.rs similarity index 86% rename from codegen/src/targets/py/mod.rs rename to codegen/src/targets/python/mod.rs index 94f14f0..31637c0 100644 --- a/codegen/src/targets/py/mod.rs +++ b/codegen/src/targets/python/mod.rs @@ -1,4 +1,4 @@ -mod c; +mod extension; mod stub; mod syntax; mod test; @@ -6,20 +6,20 @@ mod wrapper; use crate::model::CompiledFunction; -use super::GeneratedFile; +use crate::output::GeneratedFile; pub(super) const C_HEADER: &str = "/* @generated by ptfkit-codegen; DO NOT EDIT. */\n"; pub(super) const WRAPPER_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n"; pub(super) struct Output { - pub(super) c_sources: Vec, + pub(super) extension: Vec, pub(super) wrappers: Vec, pub(super) tests: Vec, } pub(super) fn render(functions: &[CompiledFunction]) -> anyhow::Result { Ok(Output { - c_sources: c::render(functions)?, + extension: extension::render(functions)?, wrappers: { let mut wrappers = wrapper::render(functions)?; wrappers.push(GeneratedFile::new( @@ -32,7 +32,7 @@ pub(super) fn render(functions: &[CompiledFunction]) -> anyhow::Result { }) } -pub(super) fn natural_sort_key(value: &str) -> String { +pub(crate) fn natural_sort_key(value: &str) -> String { let mut key = String::new(); let mut digits = String::new(); for character in value.chars() { diff --git a/codegen/src/targets/py/stub.rs b/codegen/src/targets/python/stub.rs similarity index 100% rename from codegen/src/targets/py/stub.rs rename to codegen/src/targets/python/stub.rs diff --git a/codegen/src/targets/py/syntax.rs b/codegen/src/targets/python/syntax.rs similarity index 93% rename from codegen/src/targets/py/syntax.rs rename to codegen/src/targets/python/syntax.rs index 8b59c42..9a5b282 100644 --- a/codegen/src/targets/py/syntax.rs +++ b/codegen/src/targets/python/syntax.rs @@ -28,10 +28,6 @@ impl Module { self.writer.write(value); } - pub(super) fn future_annotations(&mut self) { - self.line("from __future__ import annotations"); - } - pub(super) fn import(&mut self, module: &str, names: impl Display) { self.line(format_args!("from {module} import {names}")); } diff --git a/codegen/src/targets/py/test.rs b/codegen/src/targets/python/test.rs similarity index 87% rename from codegen/src/targets/py/test.rs rename to codegen/src/targets/python/test.rs index a5a600d..d4bb4a6 100644 --- a/codegen/src/targets/py/test.rs +++ b/codegen/src/targets/python/test.rs @@ -1,8 +1,11 @@ use std::collections::BTreeMap; -use crate::model::{CompiledFunction, Function, Outputs, PythonGeneration}; +use crate::{ + model::{CompiledFunction, Function, Outputs, PythonGeneration}, + output::GeneratedFile, +}; -use super::{super::GeneratedFile, WRAPPER_HEADER, natural_sort_key, syntax::Module}; +use super::{WRAPPER_HEADER, natural_sort_key, syntax::Module}; pub(super) fn render(functions: &[CompiledFunction]) -> Vec { let mut modules: BTreeMap> = BTreeMap::new(); @@ -40,11 +43,7 @@ fn module_source(slug: &str, functions: &[&CompiledFunction]) -> String { imports.sort_by_key(|name| natural_sort_key(name)); imports.dedup(); let mut module = Module::new(WRAPPER_HEADER); - module.line(""); - module.future_annotations(); - module.blank_line(); - module.line("import pytest"); - module.blank_line(); + module.write("\nfrom __future__ import annotations\n\nimport pytest\n\n"); module.import("_helpers", "prepare_vector_case"); module.import(&format!("ptfkit.{slug}"), imports.join(", ")); module.blank_line(); @@ -72,9 +71,7 @@ fn function_source(module: &mut Module, function: &Function) { float(case.atol), )); } - module.line("]"); - module.blank_line(); - module.blank_line(); + module.write("]\n\n\n"); let name = &function.public_api.name; module.line(format_args!( "@pytest.mark.parametrize(('inputs', 'expected', 'rtol', 'atol'), {cases_name})" @@ -82,8 +79,7 @@ fn function_source(module: &mut Module, function: &Function) { module.line(format_args!( "def test_{name}_golden(inputs: dict[str, float], expected: dict[str, float], rtol: float, atol: float):" )); - module.line(format_args!(" result = {name}(**inputs)")); - module.line(""); + module.write(format_args!(" result = {name}(**inputs)\n\n")); module.line(expected_assertion(function, " ", "")); if !function.golden_tests.is_empty() { vector_test_source(module, function, &cases_name); @@ -106,13 +102,9 @@ fn vector_test_source(module: &mut Module, function: &Function, cases_name: &str module.blank_line(); module.blank_line(); module.line(format_args!("def test_{name}_array():")); - module.line(format_args!( - " inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls})" + module.write(format_args!( + " inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls})\n result = {name}(**inputs, out=None)\n{array_assertion}\n\n\n" )); - module.line(format_args!(" result = {name}(**inputs, out=None)")); - module.line(&array_assertion); - module.blank_line(); - module.blank_line(); module.line(format_args!("def test_{name}_out():")); module.line(format_args!( " inputs, expected, rtol, atol, out = prepare_vector_case({cases_name}{result_cls})" diff --git a/codegen/src/targets/py/wrapper.rs b/codegen/src/targets/python/wrapper.rs similarity index 97% rename from codegen/src/targets/py/wrapper.rs rename to codegen/src/targets/python/wrapper.rs index 82731ae..95c7f5f 100644 --- a/codegen/src/targets/py/wrapper.rs +++ b/codegen/src/targets/python/wrapper.rs @@ -5,13 +5,10 @@ use anyhow::{Result, bail}; use crate::{ documentation::{self as docs, FunctionDocument}, model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}, + output::GeneratedFile, }; -use super::{ - super::{GeneratedFile, documentation}, - WRAPPER_HEADER, natural_sort_key, - syntax::Module, -}; +use super::{WRAPPER_HEADER, natural_sort_key, syntax::Module}; struct PythonFunction<'a> { name: &'a str, @@ -130,12 +127,9 @@ fn module_source( }); } module.blank_line(); - module.write(module_docstring(source, scope)); - module.blank_line(); - module.future_annotations(); - module.blank_line(); - module.import("typing", typing_imports); - module.blank_line(); + module.write(format_args!("{}\n\n", module_docstring(source, scope))); + module.write("from __future__ import annotations\n\n"); + module.write(format_args!("from typing import {typing_imports}\n\n")); module.block( "from ptfkit._ptfkit import (", |writer| { @@ -148,8 +142,7 @@ fn module_source( }, ")", ); - module.blank_line(); - module.blank_line(); + module.write("\n\n"); module.line("if TYPE_CHECKING:"); module.indented(|writer| { writer.line("from numpy import floating"); @@ -356,7 +349,7 @@ fn result_class_docstring(function: &Function) -> String { } fn parameter_documentation(parameter: &Parameter) -> String { - documentation::parameter_documentation(parameter) + docs::parameter_documentation(parameter) } fn definition_list_block(lines: &mut Vec, name: &str, text: &str) { diff --git a/codegen/src/targets/c_documentation.rs b/codegen/src/targets/reference/c.rs similarity index 81% rename from codegen/src/targets/c_documentation.rs rename to codegen/src/targets/reference/c.rs index e729aff..2f7f24e 100644 --- a/codegen/src/targets/c_documentation.rs +++ b/codegen/src/targets/reference/c.rs @@ -2,20 +2,16 @@ use std::{collections::BTreeSet, path::PathBuf}; use anyhow::Result; +use crate::render::markdown::HEADER; use crate::{ documentation::{self as docs}, model::{CompiledFunction, Output, Parameter}, - render::Writer, + output::GeneratedFile, + render::{Writer, markdown}, + targets::{group_by_source, native::c_result_name}, }; -use super::{ - GeneratedFile, - documentation::{self as markdown, HEADER}, - group_by_source, - native::c_result_name, -}; - -pub(super) fn render(functions: &[CompiledFunction]) -> Result> { +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { let sources = group_by_source(functions); let mut files = vec![markdown::markdown_file("index.md", |writer| { render_index(writer, &sources); @@ -64,14 +60,8 @@ fn render_index( markdown::generated_frontmatter(writer, |writer| { writer.line("title: C API reference"); }); - writer.line("# C API reference"); - writer.blank_line(); - writer.line("ptfkit's C API is organized around installed headers."); - writer.blank_line(); - writer.line("## Headers"); - writer.blank_line(); - writer.line( - "- [``](headers/ptfkit.md) — Aggregates every ptfkit source header.", + writer.write( + "# C API reference\n\nptfkit's C API is organized around installed headers.\n\n## Headers\n\n- [``](headers/ptfkit.md) — Aggregates every ptfkit source header.\n", ); for (slug, functions) in sources { let summary = docs::for_source( @@ -93,15 +83,13 @@ fn render_umbrella( sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, ) { writer.write(HEADER); - writer.line("# ``"); - writer.blank_line(); + writer.write("# ``\n\n"); markdown::code_block(writer, "c", |writer| { writer.line("#include "); }); - writer.line("This umbrella header aggregates every public ptfkit source header. Include an individual header when only one source is needed."); - writer.blank_line(); - writer.line("## Included headers"); - writer.blank_line(); + writer.write( + "This umbrella header aggregates every public ptfkit source header. Include an individual header when only one source is needed.\n\n## Included headers\n\n", + ); for (slug, functions) in sources { writer.line(format_args!( "- [``]({slug}.md) — {}", @@ -122,41 +110,37 @@ fn render_header(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction .expect("compiled source contains at least one function"); let source = docs::for_source(&first.entry.spec.source, &first.entry.spec.scope); writer.write(HEADER); - writer.line(format_args!("# ``")); - writer.blank_line(); + writer.write(format_args!("# ``\n\n")); markdown::code_block(writer, "c", |writer| { writer.line(format_args!("#include ")); }); - writer.line(escape_text(source.summary)); - writer.blank_line(); - writer.line("## Source"); - writer.blank_line(); - writer.line(escape_text(source.reference.citation)); - writer.blank_line(); + writer.write(format_args!( + "{}\n\n## Source\n\n{}\n\n", + escape_text(source.summary), + escape_text(source.reference.citation), + )); if let Some(doi) = source.reference.doi { - writer.line(format_args!( - "[DOI: {}]({})", + writer.write(format_args!( + "[DOI: {}]({})\n\n", escape_text(doi.identifier), doi.url )); - writer.blank_line(); } if source.territory.is_some() || source.dataset.is_some() { - writer.line("## Scope"); - writer.blank_line(); + writer.write("## Scope\n\n"); if let Some(territory) = source.territory { - writer.line(format_args!("**Territory:** {}", escape_text(territory))); - writer.blank_line(); + writer.write(format_args!( + "**Territory:** {}\n\n", + escape_text(territory) + )); } if let Some(dataset) = source.dataset { - writer.line(format_args!("**Dataset:** {}", escape_text(dataset))); - writer.blank_line(); + writer.write(format_args!("**Dataset:** {}\n\n", escape_text(dataset))); } } - writer.line(format_args!( - "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)" + writer.write(format_args!( + "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" )); - writer.blank_line(); let mut structures = BTreeSet::new(); for function in functions { @@ -171,8 +155,7 @@ fn render_header(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction } } } - writer.line("## Functions"); - writer.blank_line(); + writer.write("## Functions\n\n"); for function in functions { render_function_documentation(writer, function)?; } @@ -180,8 +163,7 @@ fn render_header(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction } fn render_structure(writer: &mut Writer, name: &str, fields: &[Parameter]) { - writer.line(format_args!("## `{name}`")); - writer.blank_line(); + writer.write(format_args!("## `{name}`\n\n")); markdown::code_block(writer, "c", |writer| { writer.line("typedef struct {"); writer.indented(|writer| { @@ -207,18 +189,16 @@ fn render_function_documentation(writer: &mut Writer, function: &CompiledFunctio let spec = spec(function); let document = docs::for_function(spec); let anchor = function_anchor(&function.core.name); - writer.line(format_args!("### `{}` {{#{anchor}}}", function.core.name)); - writer.blank_line(); - writer.line(escape_text(document.summary)); - writer.blank_line(); + writer.write(format_args!( + "### `{}` {{#{anchor}}}\n\n{}\n\n", + function.core.name, + escape_text(document.summary), + )); let signature = signature(function)?; markdown::code_block(writer, "c", |writer| { writer.line(format_args!("{signature};")); }); - writer.line("#### Parameters"); - writer.blank_line(); - writer.line("| Name | Direction | Description |"); - writer.line("| --- | --- | --- |"); + writer.write("#### Parameters\n\n| Name | Direction | Description |\n| --- | --- | --- |\n"); for parameter in document.parameters { writer.line(format_args!( "| `{}` | in | {} |", @@ -227,19 +207,16 @@ fn render_function_documentation(writer: &mut Writer, function: &CompiledFunctio )); } writer.blank_line(); - writer.line("#### Returns"); - writer.blank_line(); + writer.write("#### Returns\n\n"); match document.returns { docs::Returns::Scalar(field) => { - writer.line(parameter_details(field)); - writer.blank_line(); + writer.write(format_args!("{}\n\n", parameter_details(field))); } docs::Returns::Record { .. } => { let name = spec .result_class() .expect("record output has a result class"); - writer.line(format_args!("A `{}` value.", c_result_name(name))); - writer.blank_line(); + writer.write(format_args!("A `{}` value.\n\n", c_result_name(name))); } } for note in document.notes { @@ -255,10 +232,7 @@ fn render_functions_index(writer: &mut Writer, functions: &[(&str, &CompiledFunc markdown::generated_frontmatter(writer, |writer| { writer.line("title: C function index"); }); - writer.line("# C function index"); - writer.blank_line(); - writer.line("| Function | Summary | Header |"); - writer.line("| --- | --- | --- |"); + writer.write("# C function index\n\n| Function | Summary | Header |\n| --- | --- | --- |\n"); for (slug, function) in functions { writer.line(format_args!( "| [`{0}`](headers/{1}.md#{2}) | {3} | [``](headers/{1}.md) |", @@ -321,7 +295,7 @@ fn escape_table(value: &str) -> String { } fn natural_sort_key(value: &str) -> String { - super::py::natural_sort_key(value) + crate::targets::python::natural_sort_key(value) } #[cfg(test)] @@ -336,7 +310,7 @@ mod tests { .expect("codegen directory has a repository parent"); let entries = crate::specs::load(root).expect("repository specifications load"); let compiled = - super::super::compile::functions(entries).expect("repository specifications compile"); + crate::compile::functions(entries).expect("repository specifications compile"); render(&compiled).expect("C documentation renders") } diff --git a/codegen/src/targets/cpp_documentation.rs b/codegen/src/targets/reference/cpp.rs similarity index 81% rename from codegen/src/targets/cpp_documentation.rs rename to codegen/src/targets/reference/cpp.rs index b9007b1..8753fd5 100644 --- a/codegen/src/targets/cpp_documentation.rs +++ b/codegen/src/targets/reference/cpp.rs @@ -5,16 +5,12 @@ use anyhow::{Result, anyhow}; use crate::{ documentation::{self as docs}, model::{CompiledFunction, Output, Parameter}, - render::Writer, + output::GeneratedFile, + render::{Writer, markdown}, + targets::group_by_source, }; -use super::{ - GeneratedFile, - documentation::{self as markdown}, - group_by_source, -}; - -pub(super) fn render(functions: &[CompiledFunction]) -> Result> { +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { let sources = group_by_source(functions); let mut files = vec![markdown::markdown_file("index.md", |writer| { render_index(writer, &sources); @@ -53,13 +49,9 @@ fn render_index( markdown::generated_frontmatter(writer, |writer| { writer.line("title: C++ API reference"); }); - writer.line("# C++ API reference"); - writer.blank_line(); - writer.line("ptfkit's C++ API is organized around C++20 modules."); - writer.blank_line(); - writer.line("## Modules"); - writer.blank_line(); - writer.line("- [`ptfkit`](modules/ptfkit.md) — Re-exports every ptfkit source module."); + writer.write( + "# C++ API reference\n\nptfkit's C++ API is organized around C++20 modules.\n\n## Modules\n\n- [`ptfkit`](modules/ptfkit.md) — Re-exports every ptfkit source module.\n", + ); for (slug, functions) in sources { writer.line(format_args!( "- [`ptfkit.{slug}`](modules/{slug}.md) — {}", @@ -84,15 +76,13 @@ fn render_umbrella( writer.line("title: C++ module ptfkit"); writer.line("nav-title: ptfkit"); }); - writer.line("# `ptfkit`"); - writer.blank_line(); + writer.write("# `ptfkit`\n\n"); markdown::code_block(writer, "cpp", |writer| { writer.line("import ptfkit;"); }); - writer.line("This umbrella module re-exports every public ptfkit source module. Import an individual module when only one source is needed."); - writer.blank_line(); - writer.line("## Re-exported modules"); - writer.blank_line(); + writer.write( + "This umbrella module re-exports every public ptfkit source module. Import an individual module when only one source is needed.\n\n## Re-exported modules\n\n", + ); for (slug, functions) in sources { writer.line(format_args!( "- [`ptfkit.{slug}`]({slug}.md) — {}", @@ -116,43 +106,37 @@ fn render_module(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction writer.line(format_args!("title: C++ module ptfkit.{slug}")); writer.line(format_args!("nav-title: ptfkit.{slug}")); }); - writer.line(format_args!("# `ptfkit.{slug}`")); - writer.blank_line(); + writer.write(format_args!("# `ptfkit.{slug}`\n\n")); markdown::code_block(writer, "cpp", |writer| { writer.line(format_args!("import ptfkit.{slug};")); }); - writer.line(format_args!("**Exported namespace:** `ptfkit::{slug}`")); - writer.blank_line(); - writer.line(escape_text(source.summary)); - writer.blank_line(); - writer.line("## Source"); - writer.blank_line(); - writer.line(escape_text(source.reference.citation)); - writer.blank_line(); + writer.write(format_args!( + "**Exported namespace:** `ptfkit::{slug}`\n\n{}\n\n## Source\n\n{}\n\n", + escape_text(source.summary), + escape_text(source.reference.citation), + )); if let Some(doi) = source.reference.doi { - writer.line(format_args!( - "[DOI: {}]({})", + writer.write(format_args!( + "[DOI: {}]({})\n\n", escape_text(doi.identifier), doi.url )); - writer.blank_line(); } if source.territory.is_some() || source.dataset.is_some() { - writer.line("## Scope"); - writer.blank_line(); + writer.write("## Scope\n\n"); if let Some(territory) = source.territory { - writer.line(format_args!("**Territory:** {}", escape_text(territory))); - writer.blank_line(); + writer.write(format_args!( + "**Territory:** {}\n\n", + escape_text(territory) + )); } if let Some(dataset) = source.dataset { - writer.line(format_args!("**Dataset:** {}", escape_text(dataset))); - writer.blank_line(); + writer.write(format_args!("**Dataset:** {}\n\n", escape_text(dataset))); } } - writer.line(format_args!( - "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)" + writer.write(format_args!( + "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" )); - writer.blank_line(); let mut structures = BTreeSet::new(); for function in functions { @@ -163,8 +147,7 @@ fn render_module(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction } } } - writer.line("## Functions"); - writer.blank_line(); + writer.write("## Functions\n\n"); for function in functions { render_function_documentation(writer, function)?; } @@ -172,8 +155,7 @@ fn render_module(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction } fn render_structure(writer: &mut Writer, name: &str, fields: &[Parameter]) { - writer.line(format_args!("## `{name}`")); - writer.blank_line(); + writer.write(format_args!("## `{name}`\n\n")); markdown::code_block(writer, "cpp", |writer| { writer.line(format_args!("struct {name} {{")); writer.indented(|writer| { @@ -199,18 +181,16 @@ fn render_function_documentation(writer: &mut Writer, function: &CompiledFunctio let spec = spec(function); let document = docs::for_function(spec); let anchor = function_anchor(&function.core.name); - writer.line(format_args!("### `{}` {{#{anchor}}}", function.core.name)); - writer.blank_line(); - writer.line(escape_text(document.summary)); - writer.blank_line(); + writer.write(format_args!( + "### `{}` {{#{anchor}}}\n\n{}\n\n", + function.core.name, + escape_text(document.summary), + )); let signature = signature(function)?; markdown::code_block(writer, "cpp", |writer| { writer.line(signature); }); - writer.line("#### Parameters"); - writer.blank_line(); - writer.line("| Name | Description |"); - writer.line("| --- | --- |"); + writer.write("#### Parameters\n\n| Name | Description |\n| --- | --- |\n"); for parameter in document.parameters { writer.line(format_args!( "| `{}` | {} |", @@ -219,16 +199,13 @@ fn render_function_documentation(writer: &mut Writer, function: &CompiledFunctio )); } writer.blank_line(); - writer.line("#### Returns"); - writer.blank_line(); + writer.write("#### Returns\n\n"); match document.returns { docs::Returns::Scalar(field) => { - writer.line(parameter_details(field)); - writer.blank_line(); + writer.write(format_args!("{}\n\n", parameter_details(field))); } docs::Returns::Record { .. } => { - writer.line(format_args!("A `{}` value.", result_class(function)?)); - writer.blank_line(); + writer.write(format_args!("A `{}` value.\n\n", result_class(function)?)); } } for note in document.notes { @@ -244,10 +221,7 @@ fn render_functions_index(writer: &mut Writer, functions: &[(&str, &CompiledFunc markdown::generated_frontmatter(writer, |writer| { writer.line("title: C++ function index"); }); - writer.line("# C++ function index"); - writer.blank_line(); - writer.line("| Function | Summary | Module |"); - writer.line("| --- | --- | --- |"); + writer.write("# C++ function index\n\n| Function | Summary | Module |\n| --- | --- | --- |\n"); for (slug, function) in functions { let qualified = format!("ptfkit::{slug}::{}", function.core.name); writer.line(format_args!( @@ -311,7 +285,7 @@ fn escape_table(value: &str) -> String { } fn natural_sort_key(value: &str) -> String { - super::py::natural_sort_key(value) + crate::targets::python::natural_sort_key(value) } #[cfg(test)] @@ -326,7 +300,7 @@ mod tests { .expect("codegen directory has a repository parent"); let entries = crate::specs::load(root).expect("repository specifications load"); let compiled = - super::super::compile::functions(entries).expect("repository specifications compile"); + crate::compile::functions(entries).expect("repository specifications compile"); render(&compiled).expect("C++ documentation renders") } diff --git a/codegen/src/targets/reference/mod.rs b/codegen/src/targets/reference/mod.rs new file mode 100644 index 0000000..b3fd5df --- /dev/null +++ b/codegen/src/targets/reference/mod.rs @@ -0,0 +1,5 @@ +//! Language API-reference documentation products. + +pub(super) mod c; +pub(super) mod cpp; +pub(super) mod python; diff --git a/codegen/src/targets/python_documentation.rs b/codegen/src/targets/reference/python.rs similarity index 90% rename from codegen/src/targets/python_documentation.rs rename to codegen/src/targets/reference/python.rs index 6677793..7119ff2 100644 --- a/codegen/src/targets/python_documentation.rs +++ b/codegen/src/targets/reference/python.rs @@ -1,15 +1,11 @@ use crate::{ model::Entry, - render::{Render, Writer}, + output::GeneratedFile, + render::{Render, Writer, markdown}, + targets::python::natural_sort_key, }; -use super::{ - GeneratedFile, - documentation::{self as markdown}, - py::natural_sort_key, -}; - -pub(super) fn render(entries: &[Entry]) -> Vec { +pub(crate) fn render(entries: &[Entry]) -> Vec { let mut entries = entries.iter().collect::>(); entries.sort_by_key(|entry| natural_sort_key(&entry.slug)); @@ -34,12 +30,9 @@ impl Render for IndexPage<'_> { markdown::generated_frontmatter(writer, |writer| { writer.line("title: Python API reference"); }); - writer.line("# Python API reference"); - writer.blank_line(); - writer.line("ptfkit's Python API is organized around public source modules."); - writer.blank_line(); - writer.line("## Modules"); - writer.blank_line(); + writer.write( + "# Python API reference\n\nptfkit's Python API is organized around public source modules.\n\n## Modules\n\n", + ); for entry in self.entries { ModuleReference { entry }.render(writer); } diff --git a/codegen/src/targets/rs.rs b/codegen/src/targets/rust.rs similarity index 98% rename from codegen/src/targets/rs.rs rename to codegen/src/targets/rust.rs index bfa6072..13a77ac 100644 --- a/codegen/src/targets/rs.rs +++ b/codegen/src/targets/rust.rs @@ -10,7 +10,7 @@ use crate::{ semantic::{self, BinaryOp, Expr, MathFunction, Number, Reference, UnaryOp}, }; -use super::{GeneratedFile, documentation, group_by_source}; +use crate::{output::GeneratedFile, targets::group_by_source}; pub(super) const HEADER: &str = "// @generated by ptfkit-codegen; DO NOT EDIT.\n"; @@ -161,7 +161,7 @@ fn output_tokens( .iter() .find(|parameter| field == parameter.name) .expect("core output field matches specification"); - let docs = doc_tokens([documentation::parameter_details(parameter)]); + let docs = doc_tokens([docs::parameter_details(parameter)]); quote!(#docs pub #field: f64) }); let values = fields.iter().map(|field| format_ident!("{field}")); @@ -222,7 +222,7 @@ fn function_doc_tokens(document: FunctionDocument<'_>) -> TokenStream { document .parameters .iter() - .map(|parameter| format!("* {}", documentation::parameter_documentation(parameter))), + .map(|parameter| format!("* {}", docs::parameter_documentation(parameter))), ); lines.extend([String::new(), "# Returns".into(), String::new()]); lines.extend(return_doc_lines(document.returns)); @@ -273,7 +273,7 @@ fn return_doc_lines(returns: Returns<'_>) -> Vec { } Returns::Scalar(output) => std::slice::from_ref(output) .iter() - .map(|parameter| format!("* {}", documentation::parameter_documentation(parameter))) + .map(|parameter| format!("* {}", docs::parameter_documentation(parameter))) .collect(), } } @@ -717,11 +717,11 @@ mod tests { }; assert_eq!( - documentation::parameter_documentation(¶meter), + docs::parameter_documentation(¶meter), "theta_s: Saturated water content. (cm^3/cm^3)" ); assert_eq!( - documentation::parameter_details(¶meter), + docs::parameter_details(¶meter), "Saturated water content. (cm^3/cm^3)" ); } diff --git a/targets/ptfkit-native/cmake/ptfkitModules.cmake b/targets/ptfkit-native/cmake/ptfkitModules.cmake index 7fe39fa..112d3d5 100644 --- a/targets/ptfkit-native/cmake/ptfkitModules.cmake +++ b/targets/ptfkit-native/cmake/ptfkitModules.cmake @@ -1,4 +1,5 @@ # @generated by ptfkit-codegen; DO NOT EDIT. + set(PTFKIT_CPP_MODULES "${CMAKE_CURRENT_LIST_DIR}/../cpp/ptfkit.cppm" "${CMAKE_CURRENT_LIST_DIR}/../cpp/ahuja1984.cppm" From 669857088b26c5311166dbf169fbe42e72b957bc Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Fri, 21 Aug 2026 12:37:06 +0300 Subject: [PATCH 13/14] refactor(codegen): initialize Writer with capacity --- codegen/src/render/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/codegen/src/render/mod.rs b/codegen/src/render/mod.rs index 55e6e3c..48e810c 100644 --- a/codegen/src/render/mod.rs +++ b/codegen/src/render/mod.rs @@ -3,6 +3,8 @@ use std::fmt::{self, Display, Write as _}; pub(crate) mod c; pub(crate) mod markdown; +const DEFAULT_CAPACITY: usize = 16 * 1024; + /// Renders a structured value into a [`Writer`]. pub(crate) trait Render { fn render(&self, writer: &mut Writer); @@ -18,8 +20,13 @@ pub(crate) struct Writer { impl Writer { pub(crate) fn new() -> Self { + Self::with_capacity(DEFAULT_CAPACITY) + } + + /// Creates a writer with space reserved for at least `capacity` bytes. + pub(crate) fn with_capacity(capacity: usize) -> Self { Self { - contents: String::new(), + contents: String::with_capacity(capacity), indentation: 0, indent: " ", at_line_start: true, From b57891efdb507a723fe9da75e8f9d1429ce4c511 Mon Sep 17 00:00:00 2001 From: Petr Tsymbarovich Date: Fri, 21 Aug 2026 13:35:25 +0300 Subject: [PATCH 14/14] refactor(codegen): finalize --- codegen/src/render/mod.rs | 39 ++-- codegen/src/targets/native.rs | 10 +- codegen/src/targets/python/extension.rs | 11 +- codegen/src/targets/python/test.rs | 103 ++++++---- codegen/src/targets/python/wrapper.rs | 251 ++++++++++++++---------- 5 files changed, 241 insertions(+), 173 deletions(-) diff --git a/codegen/src/render/mod.rs b/codegen/src/render/mod.rs index 48e810c..810e3c4 100644 --- a/codegen/src/render/mod.rs +++ b/codegen/src/render/mod.rs @@ -39,26 +39,29 @@ impl Writer { self } - /// Writes a text fragment, adding indentation only at the start of its lines. + /// Writes one text fragment, adding indentation only at its start. /// - /// Keep large static templates as single fragments. Use [`Self::line`] and - /// [`Self::indented`] for dynamic or nested structure; templates must not - /// include their own leading indentation for a surrounding block. + /// Keep large static templates as single fragments outside an indented + /// block. Within [`Self::indented`], write each line separately with + /// [`Self::line`] or [`Self::blank_line`]; this method does not inspect + /// embedded newlines. pub(crate) fn write(&mut self, value: impl Display) { self.write_fmt(format_args!("{value}")) .expect("writing to a String cannot fail"); } pub(crate) fn line(&mut self, value: impl Display) { - self.write(value); - self.write("\n"); + self.write_fmt(format_args!("{value}\n")) + .expect("writing to a String cannot fail"); } pub(crate) fn blank_line(&mut self) { - if !self.at_line_start { - self.write("\n"); - } - self.write("\n"); + let res = if self.at_line_start { + self.write_char('\n') + } else { + self.write_str("\n\n") + }; + res.expect("writing to a String cannot fail"); } /// Renders a nested block with one additional indentation level. @@ -75,15 +78,17 @@ impl Writer { impl fmt::Write for Writer { fn write_str(&mut self, text: &str) -> fmt::Result { - for part in text.split_inclusive('\n') { - if self.at_line_start && part != "\n" { - for _ in 0..self.indentation { - self.contents.push_str(self.indent); - } + debug_assert!( + self.indentation == 0 || !text.contains('\n') || text == "\n", + "write each indented line separately" + ); + if self.at_line_start && !text.is_empty() && !text.starts_with('\n') { + for _ in 0..self.indentation { + self.contents.push_str(self.indent); } - self.contents.push_str(part); - self.at_line_start = part.ends_with('\n'); } + self.contents.push_str(text); + self.at_line_start = text.ends_with('\n'); Ok(()) } } diff --git a/codegen/src/targets/native.rs b/codegen/src/targets/native.rs index 6cb33da..bb59a0c 100644 --- a/codegen/src/targets/native.rs +++ b/codegen/src/targets/native.rs @@ -292,15 +292,19 @@ impl NativeFunction<'_> { } Output::Scalar => writer.line(format_args!("return {output_name};")), Output::Struct(fields) if matches!(self.dialect, NativeDialect::C) => { - writer.write(format_args!("#ifdef __cplusplus\nreturn {}{{", self.result)); + writer.line("#ifdef __cplusplus"); + writer.write(format_args!("return {}{{", self.result)); render_values(writer, fields); - writer.write(format_args!("}};\n#else\nreturn ({}) {{\n", self.result)); + writer.line("};"); + writer.line("#else"); + writer.line(format_args!("return ({}) {{", self.result)); writer.indented(|writer| { for field in fields { writer.line(format_args!(".{field} = {field},")); } }); - writer.write("};\n#endif\n"); + writer.line("};"); + writer.line("#endif"); } Output::Struct(fields) => { writer.write(format_args!("return {}{{", self.result)); diff --git a/codegen/src/targets/python/extension.rs b/codegen/src/targets/python/extension.rs index 1599f15..e65e4fb 100644 --- a/codegen/src/targets/python/extension.rs +++ b/codegen/src/targets/python/extension.rs @@ -56,13 +56,10 @@ pub(crate) fn render(functions: &[CompiledFunction]) -> Result String { imports.sort_by_key(|name| natural_sort_key(name)); imports.dedup(); let mut module = Module::new(WRAPPER_HEADER); - module.write("\nfrom __future__ import annotations\n\nimport pytest\n\n"); + module.line("\nfrom __future__ import annotations"); + module.blank_line(); + module.line("import pytest"); + module.blank_line(); module.import("_helpers", "prepare_vector_case"); module.import(&format!("ptfkit.{slug}"), imports.join(", ")); module.blank_line(); @@ -62,16 +65,20 @@ fn module_source(slug: &str, functions: &[&CompiledFunction]) -> String { fn function_source(module: &mut Module, function: &Function) { let cases_name = format!("CASES_{}", function.public_api.name.to_ascii_uppercase()); module.assignment(&cases_name, "["); - for case in &function.golden_tests { - module.line(format_args!( - " ({}, {}, {}, {}),", - dictionary(&case.inputs), - dictionary(&case.expected), - float(case.rtol), - float(case.atol), - )); - } - module.write("]\n\n\n"); + module.indented(|writer| { + for case in &function.golden_tests { + writer.line(format_args!( + "({}, {}, {}, {}),", + dictionary(&case.inputs), + dictionary(&case.expected), + float(case.rtol), + float(case.atol), + )); + } + }); + module.line("]"); + module.blank_line(); + module.blank_line(); let name = &function.public_api.name; module.line(format_args!( "@pytest.mark.parametrize(('inputs', 'expected', 'rtol', 'atol'), {cases_name})" @@ -79,21 +86,17 @@ fn function_source(module: &mut Module, function: &Function) { module.line(format_args!( "def test_{name}_golden(inputs: dict[str, float], expected: dict[str, float], rtol: float, atol: float):" )); - module.write(format_args!(" result = {name}(**inputs)\n\n")); - module.line(expected_assertion(function, " ", "")); + module.indented(|writer| { + writer.line(format_args!("result = {name}(**inputs)")); + writer.blank_line(); + render_expected_assertion(writer, function, ""); + }); if !function.golden_tests.is_empty() { vector_test_source(module, function, &cases_name); } } fn vector_test_source(module: &mut Module, function: &Function, cases_name: &str) { - let array_assertion = expected_assertion(function, " ", "[0]"); - let out_assertion = match &function.outputs { - Outputs::Scalar { .. } => " assert result is out", - Outputs::Record { .. } => { - " for actual, expected_out in zip(result, out, strict=True):\n assert actual is expected_out" - } - }; let result_cls = function .result_class() .map(|result_class| format!(", {result_class}")) @@ -102,34 +105,50 @@ fn vector_test_source(module: &mut Module, function: &Function, cases_name: &str module.blank_line(); module.blank_line(); module.line(format_args!("def test_{name}_array():")); - module.write(format_args!( - " inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls})\n result = {name}(**inputs, out=None)\n{array_assertion}\n\n\n" - )); + module.indented(|writer| { + writer.line(format_args!( + "inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls})" + )); + writer.line(format_args!("result = {name}(**inputs, out=None)")); + render_expected_assertion(writer, function, "[0]"); + }); + module.blank_line(); + module.blank_line(); module.line(format_args!("def test_{name}_out():")); - module.line(format_args!( - " inputs, expected, rtol, atol, out = prepare_vector_case({cases_name}{result_cls})" - )); - module.line(format_args!(" result = {name}(**inputs, out=out)")); - module.line(out_assertion); - module.line(array_assertion); + module.indented(|writer| { + writer.line(format_args!( + "inputs, expected, rtol, atol, out = prepare_vector_case({cases_name}{result_cls})" + )); + writer.line(format_args!("result = {name}(**inputs, out=out)")); + render_out_assertion(writer, function); + render_expected_assertion(writer, function, "[0]"); + }); } -fn expected_assertion(function: &Function, indent: &str, index: &str) -> String { +fn render_expected_assertion(writer: &mut crate::render::Writer, function: &Function, index: &str) { match &function.outputs { - Outputs::Scalar { field } => format!( - "{indent}assert result{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", + Outputs::Scalar { field } => writer.line(format_args!( + "assert result{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", field.name - ), - Outputs::Record { fields, .. } => fields - .iter() - .map(|field| { - format!( - "{indent}assert result.{}{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", + )), + Outputs::Record { fields, .. } => { + for field in fields { + writer.line(format_args!( + "assert result.{}{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", field.name, field.name - ) - }) - .collect::>() - .join("\n"), + )); + } + } + } +} + +fn render_out_assertion(writer: &mut crate::render::Writer, function: &Function) { + match &function.outputs { + Outputs::Scalar { .. } => writer.line("assert result is out"), + Outputs::Record { .. } => { + writer.line("for actual, expected_out in zip(result, out, strict=True):"); + writer.indented(|writer| writer.line("assert actual is expected_out")); + } } } diff --git a/codegen/src/targets/python/wrapper.rs b/codegen/src/targets/python/wrapper.rs index 95c7f5f..3a08d34 100644 --- a/codegen/src/targets/python/wrapper.rs +++ b/codegen/src/targets/python/wrapper.rs @@ -6,10 +6,13 @@ use crate::{ documentation::{self as docs, FunctionDocument}, model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}, output::GeneratedFile, + render::Writer, }; use super::{WRAPPER_HEADER, natural_sort_key, syntax::Module}; +const LINE_WIDTH: usize = 100; + struct PythonFunction<'a> { name: &'a str, rust_name: &'a str, @@ -18,13 +21,19 @@ struct PythonFunction<'a> { array_inputs: Vec, keyword_inputs: Vec, parameters: String, - docstring: String, + docstring: PythonDocstring, +} + +#[derive(PartialEq, Eq)] +struct PythonDocstring { + summary: String, + sections: Vec<(&'static str, Vec)>, } struct PythonResultClass { name: String, field_definitions: String, - docstring: String, + docstring: PythonDocstring, } pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { @@ -54,7 +63,7 @@ pub(crate) fn render(functions: &[CompiledFunction]) -> Result>() .join("\n"), docstring: result_class_docstring(function), @@ -117,7 +126,7 @@ fn module_source( ) .chars() .count() - > 100 + > LINE_WIDTH }); module.blank_line(); module.line(if has_long_import { @@ -127,9 +136,10 @@ fn module_source( }); } module.blank_line(); - module.write(format_args!("{}\n\n", module_docstring(source, scope))); - module.write("from __future__ import annotations\n\n"); - module.write(format_args!("from typing import {typing_imports}\n\n")); + render_module_docstring(&mut module, source, scope); + module.write(format_args!( + "from __future__ import annotations\n\nfrom typing import {typing_imports}\n\n" + )); module.block( "from ptfkit._ptfkit import (", |writer| { @@ -142,8 +152,7 @@ fn module_source( }, ")", ); - module.write("\n\n"); - module.line("if TYPE_CHECKING:"); + module.line("\n\nif TYPE_CHECKING:"); module.indented(|writer| { writer.line("from numpy import floating"); writer.line("from numpy.typing import ArrayLike, NDArray"); @@ -158,10 +167,10 @@ fn module_source( class.name )); module.indented(|writer| { - writer.write(&class.docstring); - writer.write("\n"); - writer.write(&class.field_definitions); - writer.write("\n"); + render_docstring(writer, &class.docstring, 4); + for line in class.field_definitions.lines() { + writer.line(line); + } }); } } @@ -180,13 +189,13 @@ fn module_source( for function in functions { module.blank_line(); module.blank_line(); - module.write(function_source(function)); + render_function(&mut module, function); } module.line(""); module.into_string() } -fn function_source(function: &PythonFunction<'_>) -> String { +fn render_function(module: &mut Module, function: &PythonFunction<'_>) { let scalar_result = function .result_class .map(|class| format!("{class}[floating]")) @@ -205,20 +214,51 @@ fn function_source(function: &PythonFunction<'_>) -> String { } else { "return values".into() }; - let calculation = format!( - " if out is None:\n values = _{}({})\n else:\n values = _{}({}, out={out})", - function.rust_name, function.parameters, function.rust_name, function.parameters - ); - format!( - "@overload\ndef {}(*, {}) -> {scalar_result}: ...\n\n@overload\ndef {}(*, {},\n out: {array_result} | None = None,\n) -> {array_result}: ...\n\ndef {}(*,\n{}\n out: {array_result} | None = None,\n) -> {scalar_result} | {array_result}:\n{}\n{calculation}\n\n {result}", + module.line("@overload"); + module.line(format_args!( + "def {}(*, {}) -> {scalar_result}: ...", function.name, function.scalar_inputs.join(", "), + )); + module.blank_line(); + module.line("@overload"); + module.line(format_args!( + "def {}(*, {},", function.name, function.array_inputs.join(", "), - function.name, - function.keyword_inputs.join("\n"), - function.docstring, - ) + )); + module.indented(|writer| { + writer.line(format_args!("out: {array_result} | None = None,")); + }); + module.line(format_args!(") -> {array_result}: ...")); + module.blank_line(); + module.line(format_args!("def {}(*,", function.name)); + module.indented(|writer| { + for input in &function.keyword_inputs { + writer.line(input); + } + writer.line(format_args!("out: {array_result} | None = None,")); + }); + module.line(format_args!(") -> {scalar_result} | {array_result}:")); + module.indented(|writer| { + render_docstring(writer, &function.docstring, 4); + writer.line("if out is None:"); + writer.indented(|writer| { + writer.line(format_args!( + "values = _{}({})", + function.rust_name, function.parameters + )); + }); + writer.line("else:"); + writer.indented(|writer| { + writer.line(format_args!( + "values = _{}({}, out={out})", + function.rust_name, function.parameters + )); + }); + writer.blank_line(); + writer.line(result); + }); } fn view(resolved: &CompiledFunction) -> PythonFunction<'_> { @@ -240,39 +280,37 @@ fn view(resolved: &CompiledFunction) -> PythonFunction<'_> { keyword_inputs: function .inputs .iter() - .map(|input| format!(" {}: float | ArrayLike,", input.name)) + .map(|input| format!("{}: float | ArrayLike,", input.name)) .collect(), parameters: names.join(", "), docstring: function_docstring(function), } } -fn module_docstring(source: &Source, scope: &Scope) -> String { +fn render_module_docstring(module: &mut Module, source: &Source, scope: &Scope) { let document = docs::for_source(source, scope); - let mut lines = vec![ - format!("r\"\"\"{}", document.summary), - String::new(), - "Reference:".into(), - ]; - lines.extend(wrap_markdown_block(document.reference.citation, " ")); + module.line(format_args!("r\"\"\"{}", document.summary)); + module.blank_line(); + module.line("Reference:"); + render_markdown_block(module, document.reference.citation, " "); if let Some(doi) = document.reference.doi { - lines.extend(wrap_markdown_block( + render_markdown_block( + module, &format!("[DOI: {}]({})", doi.identifier, doi.url), " ", - )); + ); } if let Some(territory) = document.territory { - definition_list_block(&mut lines, "Territory", territory); + render_definition_list_block(module, "Territory", territory); } if let Some(dataset) = document.dataset { - definition_list_block(&mut lines, "Dataset", dataset); + render_definition_list_block(module, "Dataset", dataset); } - lines.push(String::new()); - lines.push("\"\"\"".into()); - lines.join("\n") + module.blank_line(); + module.line("\"\"\""); } -fn function_docstring(function: &Function) -> String { +fn function_docstring(function: &Function) -> PythonDocstring { let document = docs::for_function(function); function_docstring_from_document(document, function.result_class()) } @@ -280,7 +318,7 @@ fn function_docstring(function: &Function) -> String { fn function_docstring_from_document( document: FunctionDocument<'_>, result_class: Option<&str>, -) -> String { +) -> PythonDocstring { let mut arguments = document .parameters .iter() @@ -328,14 +366,16 @@ fn function_docstring_from_document( if !document.warnings.is_empty() { sections.push(("Warning", document.warnings.to_vec())); } - render_docstring(" ", document.summary, §ions, 4) + PythonDocstring { + summary: document.summary.to_owned(), + sections, + } } -fn result_class_docstring(function: &Function) -> String { - render_docstring( - " ", - "Results returned by the matching PTF.", - &[( +fn result_class_docstring(function: &Function) -> PythonDocstring { + PythonDocstring { + summary: "Results returned by the matching PTF.".into(), + sections: vec![( "Attributes", function .outputs @@ -344,28 +384,27 @@ fn result_class_docstring(function: &Function) -> String { .map(parameter_documentation) .collect(), )], - 4, - ) + } } fn parameter_documentation(parameter: &Parameter) -> String { docs::parameter_documentation(parameter) } -fn definition_list_block(lines: &mut Vec, name: &str, text: &str) { - lines.push(String::new()); - lines.push(name.into()); - lines.push(String::new()); - let wrapped = wrap_markdown_block(text, ": "); - lines.extend(wrapped); +fn render_definition_list_block(module: &mut Module, name: &str, text: &str) { + module.blank_line(); + module.line(name); + module.blank_line(); + render_markdown_block(module, text, ": "); } -fn wrap_markdown_block(text: &str, first_prefix: &str) -> Vec { - let first_width = 100 - first_prefix.len(); +fn render_markdown_block(module: &mut Module, text: &str, first_prefix: &str) { + let first_width = LINE_WIDTH - first_prefix.len(); let wrapped = wrap_doc_line(text, first_width, 96); - let mut lines = vec![format!("{first_prefix}{}", wrapped[0])]; - lines.extend(wrapped.iter().skip(1).map(|line| format!(" {line}"))); - lines + module.line(format_args!("{first_prefix}{}", wrapped[0])); + for line in wrapped.iter().skip(1) { + module.line(format_args!(" {line}")); + } } fn with_terminal_punctuation(text: &str) -> String { @@ -376,64 +415,52 @@ fn with_terminal_punctuation(text: &str) -> String { } } -fn render_docstring( - indent: &str, - summary: &str, - sections: &[(&str, Vec)], - indent_width: usize, -) -> String { - let raw = summary.contains('\\') - || sections +fn render_docstring(writer: &mut Writer, docstring: &PythonDocstring, indent_width: usize) { + let left_width = LINE_WIDTH - indent_width; + let raw = docstring.summary.contains('\\') + || docstring + .sections .iter() .flat_map(|(title, entries)| { std::iter::once(*title).chain(entries.iter().map(String::as_str)) }) .any(|text| text.contains('\\')); let opening = if raw { "r\"\"\"" } else { "\"\"\"" }; - let summary = with_terminal_punctuation(summary); - let first_line_width = 100 - indent_width - 3; + let summary = with_terminal_punctuation(&docstring.summary); + let first_line_width = left_width - 3; let (summary, description) = if summary.len() <= first_line_width { (summary, None) } else { ( "Calculate the pedotransfer function.".into(), - Some(wrap_doc_line( - &summary, - 100 - indent_width, - 100 - indent_width, - )), + Some(wrap_doc_line(&summary, left_width, left_width)), ) }; - let mut lines = vec![format!("{indent}{opening}{summary}"), String::new()]; + writer.line(format_args!("{opening}{summary}")); + writer.blank_line(); if let Some(description) = description { - lines.extend( - description - .into_iter() - .map(|line| format!("{indent}{line}")), - ); - lines.push(String::new()); + for line in description { + writer.line(line); + } + writer.blank_line(); } - for (section_index, (title, entries)) in sections.iter().enumerate() { + for (section_index, (title, entries)) in docstring.sections.iter().enumerate() { if !title.is_empty() { - lines.push(format!("{indent}{title}:")); + writer.line(format_args!("{title}:")); } for entry in entries { - let wrapped = wrap_doc_line(entry, 100 - indent_width - 4, 100 - indent_width - 8); - lines.push(format!("{indent} {}", wrapped[0])); - lines.extend( - wrapped - .iter() - .skip(1) - .map(|line| format!("{indent} {line}")), - ); + let wrapped = wrap_doc_line(entry, left_width - 4, left_width - 8); + writer.line(format_args!(" {}", wrapped[0])); + for line in wrapped.iter().skip(1) { + writer.line(format_args!(" {line}")); + } } - if section_index + 1 != sections.len() { - lines.push(String::new()); + if section_index + 1 != docstring.sections.len() { + writer.blank_line(); } } - lines.push(String::new()); - lines.push(format!("{indent}\"\"\"")); - lines.join("\n") + writer.blank_line(); + writer.line("\"\"\""); } fn wrap_doc_line(text: &str, first_width: usize, continuation_width: usize) -> Vec { @@ -459,8 +486,11 @@ fn wrap_doc_line(text: &str, first_width: usize, continuation_width: usize) -> V #[cfg(test)] mod tests { - use super::{function_docstring, module_docstring}; - use crate::model::{Documentation, Function, FunctionScope, Models, PublicApi, Scope, Source}; + use super::{function_docstring, render_module_docstring}; + use crate::{ + model::{Documentation, Function, FunctionScope, Models, PublicApi, Scope, Source}, + targets::python::syntax::Module, + }; fn function(territory: Option<&str>) -> Function { Function { @@ -489,11 +519,22 @@ mod tests { #[test] fn function_territory_is_rendered_only_when_declared() { + let with_territory = function_docstring(&function(Some("Narrow test region."))); + assert_eq!( + with_territory + .sections + .iter() + .find(|(title, _)| *title == "Territory") + .map(|(_, entries)| entries), + Some(&vec!["Narrow test region.".into()]) + ); + let without_territory = function_docstring(&function(None)); assert!( - function_docstring(&function(Some("Narrow test region."))) - .contains("Territory:\n Narrow test region.") + without_territory + .sections + .iter() + .all(|(title, _)| *title != "Territory") ); - assert!(!function_docstring(&function(None)).contains("Territory:")); } #[test] @@ -511,7 +552,9 @@ mod tests { dataset: None, }; - let docstring = module_docstring(&source, &scope); + let mut module = Module::new(""); + render_module_docstring(&mut module, &source, &scope); + let docstring = module.into_string(); assert_eq!( docstring.lines().next(), Some("r\"\"\"Test et al. (2026), short territory.")