diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9c8828d..aa9a8b8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -71,10 +71,7 @@ jobs: run: uv sync --project targets/ptfkit-py --frozen --all-groups - name: Check generated files - run: | - cargo run --manifest-path codegen/Cargo.toml generate - git diff --exit-code - test -z "$(git status --porcelain)" + run: cargo run --manifest-path codegen/Cargo.toml check-generated - name: Check codegen formatting id: codegen-format diff --git a/Justfile b/Justfile index bfc7130..ef23761 100644 --- a/Justfile +++ b/Justfile @@ -18,6 +18,11 @@ mod rust 'targets/ptfkit-rs' @generate: cargo run generate +# Regenerate all codegen-owned files and fail if any output drifts. +[working-directory: 'codegen'] +@check-generated: + cargo run check-generated + # Set the package version and refresh dependent lockfiles. @version value: cargo run --manifest-path codegen/Cargo.toml -- version {{quote(value)}} diff --git a/codegen/README.md b/codegen/README.md new file mode 100644 index 0000000..0dc31fd --- /dev/null +++ b/codegen/README.md @@ -0,0 +1,32 @@ +# Code Generator Architecture + +`ptfkit-codegen` validates the YAML specifications, compiles their formulas +into a shared semantic representation, and renders every committed target. +The generated files, after their target formatter runs, are the compatibility +contract; internal generator APIs are not. + +## Pipeline + +1. `specs` loads source specifications and `validate` checks their contracts. +2. `compile` resolves formulas and golden cases into `CompiledFunction` values. +3. `documentation` provides borrowed source/function facts without target + markup. `render` contains shared text, Markdown, and C-family expression + rendering support. +4. `targets::{catalog, reference, native, python, rust}` render concrete + generated products and return `GeneratedFile` artifacts. +5. `output` owns layouts, staging, formatter execution, cleanup, snapshots, + and atomic replacement. + +`check-generated` snapshots all marker-owned files, runs the same pipeline, +and fails if the generated tree changes. + +## Extension points + +Add target-local syntax and escaping beside its renderer. Reusable C-family +expression precedence belongs in `render::c`; reusable Markdown file/block +composition belongs in `render::markdown`. Use `render::Writer` only where +indentation-aware text composition is natural; Rust remains token-based with +`proc_macro2` and `quote`. Add an `output::Layout` only when a new output group +needs a distinct root, cleanup root, marker, or formatter. Do not introduce a +cross-language syntax AST or let renderers own filesystem, staging, formatting, +or cleanup policy. diff --git a/codegen/src/targets/compile.rs b/codegen/src/compile.rs similarity index 100% rename from codegen/src/targets/compile.rs rename to codegen/src/compile.rs diff --git a/codegen/src/documentation.rs b/codegen/src/documentation.rs new file mode 100644 index 0000000..e912483 --- /dev/null +++ b/codegen/src/documentation.rs @@ -0,0 +1,201 @@ +//! Semantic documentation assembled from validated specifications. +//! +//! Targets choose their own section ordering and markup. This module only +//! describes the information they have available to render. + +use crate::model::{Function, Outputs, Parameter, Scope, Source}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct SourceDocument<'a> { + pub(crate) summary: &'a str, + pub(crate) reference: Reference<'a>, + pub(crate) territory: Option<&'a str>, + pub(crate) dataset: Option<&'a str>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Reference<'a> { + pub(crate) citation: &'a str, + pub(crate) doi: Option>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Doi<'a> { + pub(crate) identifier: &'a str, + pub(crate) url: &'a str, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct FunctionDocument<'a> { + pub(crate) summary: &'a str, + pub(crate) parameters: &'a [Parameter], + pub(crate) returns: Returns<'a>, + pub(crate) territory: Option<&'a str>, + pub(crate) models: Models<'a>, + pub(crate) remarks: Remarks<'a>, + pub(crate) notes: &'a [String], + pub(crate) warnings: &'a [String], +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Returns<'a> { + Scalar(&'a Parameter), + Record { + name: &'a str, + fields: &'a [Parameter], + }, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Models<'a> { + pub(crate) h_theta: Option<&'a str>, + pub(crate) k_h: Option<&'a str>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Remarks<'a> { + pub(crate) prediction_target: &'a str, +} + +pub(crate) fn for_source<'a>(source: &'a Source, scope: &'a Scope) -> SourceDocument<'a> { + SourceDocument { + summary: &source.summary, + reference: Reference { + citation: &source.citation_apa, + doi: source.doi.as_ref().map(|doi| Doi { + identifier: &doi.identifier, + url: &doi.url, + }), + }, + territory: scope.territory.as_deref(), + dataset: scope.dataset.as_deref(), + } +} + +pub(crate) fn for_function(function: &Function) -> FunctionDocument<'_> { + FunctionDocument { + summary: &function.public_api.summary, + parameters: &function.inputs, + returns: match &function.outputs { + Outputs::Scalar { field } => Returns::Scalar(field), + Outputs::Record { name, fields } => Returns::Record { name, fields }, + }, + territory: function.scope.territory.as_deref(), + models: Models { + h_theta: function.scope.models.h_theta.as_deref(), + k_h: function.scope.models.k_h.as_deref(), + }, + remarks: Remarks { + prediction_target: &function.scope.prediction_target, + }, + notes: &function.documentation.notes, + warnings: &function.documentation.warnings, + } +} + +pub(crate) fn parameter_details(parameter: &Parameter) -> String { + format!("{} ({})", parameter.description, parameter.unit) +} + +pub(crate) fn parameter_documentation(parameter: &Parameter) -> String { + format!("{}: {}", parameter.name, parameter_details(parameter)) +} + +#[cfg(test)] +mod tests { + use crate::model::{ + Documentation, Function, FunctionScope, Models, Outputs, Parameter, PublicApi, + }; + + use super::{Returns, for_function}; + + fn parameter(name: &str) -> Parameter { + Parameter { + name: name.into(), + unit: "cm^3/cm^3".into(), + domain: None, + description: format!("{name} description."), + } + } + + #[test] + fn preserves_empty_optional_documentation_sections() { + let function = Function { + name: "calc_ptf_test".into(), + status: "draft".into(), + public_api: PublicApi { + name: "calc_ptf_test".into(), + result_class: None, + summary: "Estimate a test property.".into(), + }, + scope: FunctionScope { + territory: None, + prediction_target: "Test property.".into(), + models: Models::default(), + }, + inputs: Vec::new(), + outputs: Outputs::Scalar { + field: parameter("result"), + }, + documentation: Documentation::default(), + implementation: None, + golden_tests: Vec::new(), + }; + + let document = for_function(&function); + assert!(document.territory.is_none()); + assert!(document.models.h_theta.is_none()); + assert!(document.models.k_h.is_none()); + assert!(document.notes.is_empty()); + assert!(document.warnings.is_empty()); + assert!(matches!(document.returns, Returns::Scalar(_))); + } + + #[test] + fn retains_each_record_output_field() { + let function = Function { + name: "calc_ptf_test".into(), + status: "draft".into(), + public_api: PublicApi { + name: "calc_ptf_test".into(), + result_class: Some("TestResult".into()), + summary: "Estimate test properties.".into(), + }, + scope: FunctionScope { + territory: Some("Test territory.".into()), + prediction_target: "Test properties.".into(), + models: Models { + h_theta: Some("Retention model.".into()), + k_h: Some("Conductivity model.".into()), + }, + }, + inputs: vec![parameter("sand")], + outputs: Outputs::Record { + name: "TestResult".into(), + fields: vec![parameter("theta_33"), parameter("theta_1500")], + }, + documentation: Documentation { + notes: vec!["A note.".into()], + warnings: vec!["A warning.".into()], + }, + implementation: None, + golden_tests: Vec::new(), + }; + + let document = for_function(&function); + let Returns::Record { name, fields } = document.returns else { + panic!("record outputs must retain their shape"); + }; + assert_eq!(name, "TestResult"); + assert_eq!( + fields + .iter() + .map(|field| field.name.as_str()) + .collect::>(), + ["theta_33", "theta_1500"] + ); + assert_eq!(document.models.h_theta, Some("Retention model.")); + assert_eq!(document.models.k_h, Some("Conductivity model.")); + assert_eq!(document.remarks.prediction_target, "Test properties."); + } +} diff --git a/codegen/src/main.rs b/codegen/src/main.rs index e4583a3..304aeef 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -5,8 +5,12 @@ use std::{path::Path, process::ExitCode}; use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; +mod compile; +mod documentation; mod formula; mod model; +mod output; +mod render; mod semantic; mod specs; mod targets; @@ -24,6 +28,7 @@ pub(crate) struct Cli { enum Command { Validate, Generate, + CheckGenerated, Version { version: String }, } @@ -50,6 +55,10 @@ impl Cli { let entries = load_validated_specifications(root)?; targets::run(root, entries) } + Command::CheckGenerated => { + let entries = load_validated_specifications(root)?; + targets::check_generated(root, entries) + } Command::Version { version } => version::run(root, &version), } } diff --git a/codegen/src/output/mod.rs b/codegen/src/output/mod.rs new file mode 100644 index 0000000..546d2e2 --- /dev/null +++ b/codegen/src/output/mod.rs @@ -0,0 +1,169 @@ +//! Generated-artifact layout, formatting, and atomic filesystem commit. + +mod write; + +use std::path::PathBuf; + +pub(crate) use write::{assert_unchanged, commit, snapshot_generated}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Formatter { + None, + Rust, + Python, + C, + Cpp, +} + +pub(crate) struct Layout { + pub(super) output_directory: &'static str, + pub(super) cleanup_directory: &'static str, + pub(super) generated_header: &'static str, + pub(super) formatter: Formatter, +} + +macro_rules! layout { + ($name:ident, $output:literal, $cleanup:literal, $header:expr, $formatter:expr) => { + pub(crate) static $name: Layout = Layout { + output_directory: $output, + cleanup_directory: $cleanup, + generated_header: $header, + formatter: $formatter, + }; + }; +} + +const MARKDOWN_HEADER: &str = "\n\n"; +const RUST_HEADER: &str = "// @generated by ptfkit-codegen; DO NOT EDIT.\n"; +const C_HEADER: &str = "/* @generated by ptfkit-codegen; DO NOT EDIT. */\n"; +const PYTHON_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n"; + +layout!( + CATALOG, + "docs/src/ptf-catalog/sources", + "docs/src/ptf-catalog/sources", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + REFERENCE_C, + "docs/src/reference/c", + "docs/src/reference/c", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + REFERENCE_CPP, + "docs/src/reference/cpp", + "docs/src/reference/cpp", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + REFERENCE_PYTHON, + "docs/src/reference/python", + "docs/src/reference/python", + MARKDOWN_HEADER, + Formatter::None +); +layout!( + RUST, + "targets/ptfkit-rs/src", + "targets/ptfkit-rs/src", + RUST_HEADER, + Formatter::Rust +); +layout!( + PYTHON_EXTENSION, + "targets/ptfkit-py", + "targets/ptfkit-py/src/ptfkit", + C_HEADER, + Formatter::C +); +layout!( + PYTHON_WRAPPER, + "targets/ptfkit-py/src", + "targets/ptfkit-py/src/ptfkit", + PYTHON_HEADER, + Formatter::Python +); +layout!( + PYTHON_TEST, + "targets/ptfkit-py", + "targets/ptfkit-py/tests", + PYTHON_HEADER, + Formatter::Python +); +layout!( + NATIVE_C, + "targets/ptfkit-native/include", + "targets/ptfkit-native/include", + C_HEADER, + Formatter::Cpp +); +layout!( + NATIVE_CPP_MODULE, + "targets/ptfkit-native/cpp", + "targets/ptfkit-native/cpp", + C_HEADER, + Formatter::Cpp +); +layout!( + NATIVE_CPP_CMAKE, + "targets/ptfkit-native/cmake", + "targets/ptfkit-native/cmake", + PYTHON_HEADER, + Formatter::None +); +layout!( + NATIVE_C_TEST, + "targets/ptfkit-native/tests/c", + "targets/ptfkit-native/tests/c", + C_HEADER, + Formatter::C +); +layout!( + NATIVE_CPP_TEST, + "targets/ptfkit-native/tests/cpp", + "targets/ptfkit-native/tests/cpp", + C_HEADER, + Formatter::Cpp +); + +pub(crate) const LAYOUTS: [&Layout; 13] = [ + &CATALOG, + &REFERENCE_C, + &REFERENCE_CPP, + &REFERENCE_PYTHON, + &RUST, + &PYTHON_EXTENSION, + &PYTHON_WRAPPER, + &PYTHON_TEST, + &NATIVE_C, + &NATIVE_CPP_MODULE, + &NATIVE_CPP_CMAKE, + &NATIVE_C_TEST, + &NATIVE_CPP_TEST, +]; + +pub(crate) struct Output { + pub(super) layout: &'static Layout, + pub(super) files: Vec, +} + +impl Output { + pub(crate) fn new(layout: &'static Layout, files: Vec) -> Self { + Self { layout, files } + } +} + +pub(crate) struct GeneratedFile { + pub(crate) path: PathBuf, + pub(crate) contents: String, +} + +impl GeneratedFile { + pub(crate) fn new(path: PathBuf, contents: String) -> Self { + Self { path, contents } + } +} diff --git a/codegen/src/targets/write.rs b/codegen/src/output/write.rs similarity index 55% rename from codegen/src/targets/write.rs rename to codegen/src/output/write.rs index a3726e1..e862c7b 100644 --- a/codegen/src/targets/write.rs +++ b/codegen/src/output/write.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeSet, + collections::{BTreeMap, BTreeSet}, fs::{self, OpenOptions}, io::Write, path::{Path, PathBuf}, @@ -9,17 +9,110 @@ use std::{ use anyhow::{Context, Result, bail}; -use super::{Target, TargetOutput}; +use super::{Formatter, LAYOUTS, Layout, Output}; struct StagedWrite { target: PathBuf, temporary: PathBuf, - output_target: Target, + layout: &'static Layout, } static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); -pub(super) fn commit(root: &Path, outputs: &[TargetOutput]) -> Result<()> { +pub(crate) struct GeneratedTree(BTreeMap>); + +pub(crate) fn snapshot_generated(root: &Path) -> Result { + let mut files = BTreeMap::new(); + for layout in LAYOUTS { + collect_generated( + root, + &root.join(layout.cleanup_directory), + layout.generated_header, + &mut files, + )?; + } + Ok(GeneratedTree(files)) +} + +pub(crate) fn assert_unchanged(root: &Path, before: GeneratedTree) -> Result<()> { + let after = snapshot_generated(root)?; + if let Some(report) = drift_report(&before.0, &after.0) { + bail!("generated output drift after regeneration:\n{report}") + } + println!("generated output is unchanged"); + Ok(()) +} + +fn collect_generated( + root: &Path, + directory: &Path, + header: &str, + files: &mut BTreeMap>, +) -> Result<()> { + if !directory.is_dir() { + return Ok(()); + } + for entry in fs::read_dir(directory)? { + let path = entry?.path(); + if path.is_dir() { + collect_generated(root, &path, header, files)?; + } else { + let contents = fs::read(&path)?; + if is_generated(&contents, header) { + let relative = path + .strip_prefix(root) + .context("finding generated-file path relative to the workspace root")? + .to_owned(); + files.insert(relative, contents); + } + } + } + Ok(()) +} + +fn drift_report( + before: &BTreeMap>, + after: &BTreeMap>, +) -> Option { + let added = after + .keys() + .filter(|path| !before.contains_key(*path)) + .collect::>(); + let removed = before + .keys() + .filter(|path| !after.contains_key(*path)) + .collect::>(); + let modified = before + .iter() + .filter(|(path, contents)| after.get(*path).is_some_and(|after| after != *contents)) + .map(|(path, _)| path) + .collect::>(); + + if added.is_empty() && removed.is_empty() && modified.is_empty() { + return None; + } + + let mut report = String::new(); + append_paths(&mut report, "added", &added); + append_paths(&mut report, "removed", &removed); + append_paths(&mut report, "modified", &modified); + Some(report.trim_end().to_owned()) +} + +fn append_paths(report: &mut String, label: &str, paths: &[&PathBuf]) { + if paths.is_empty() { + return; + } + report.push_str(label); + report.push_str(":\n"); + for path in paths { + report.push_str(" "); + report.push_str(&path.display().to_string()); + report.push('\n'); + } +} + +pub(crate) fn commit(root: &Path, outputs: &[Output]) -> Result<()> { let staged = stage(root, outputs)?; if let Err(error) = format(root, &staged) { remove_temporary(&staged); @@ -37,14 +130,14 @@ pub(super) fn commit(root: &Path, outputs: &[TargetOutput]) -> Result<()> { Ok(()) } -fn stage(root: &Path, outputs: &[TargetOutput]) -> Result> { +fn stage(root: &Path, outputs: &[Output]) -> Result> { let mut staged = Vec::new(); for output in outputs { for file in &output.files { - let target = output.target.output_path(root, &file.path); + let target = root.join(output.layout.output_directory).join(&file.path); if target.exists() && fs::read_to_string(&target)? == file.contents - && !output.target.is_clang_formatted() + && output.layout.formatter == Formatter::None { continue; } @@ -59,7 +152,7 @@ fn stage(root: &Path, outputs: &[TargetOutput]) -> Result> { staged.push(StagedWrite { target, temporary, - output_target: output.target, + layout: output.layout, }); } } @@ -89,16 +182,16 @@ fn write_temporary(path: &Path, contents: &str) -> Result<()> { .with_context(|| format!("writing temporary generated file {}", path.display())) } -fn cleanup(root: &Path, output: &TargetOutput) -> Result<()> { +fn cleanup(root: &Path, output: &Output) -> Result<()> { let expected = output .files .iter() - .map(|file| output.target.output_path(root, &file.path)) + .map(|file| root.join(output.layout.output_directory).join(&file.path)) .collect::>(); remove_obsolete( - &output.target.cleanup_directory(root), + &root.join(output.layout.cleanup_directory), &expected, - output.target.generated_header(), + output.layout.generated_header, ) } @@ -119,27 +212,39 @@ fn remove_obsolete(directory: &Path, expected: &BTreeSet, header: &str) fn is_generated(contents: &[u8], header: &str) -> bool { contents.starts_with(header.as_bytes()) - || (header == super::documentation::HEADER - && contents.starts_with(b"---\n") - && contents - .windows(header.len()) - .any(|window| window == header.as_bytes())) + || (header == super::MARKDOWN_HEADER + && (contents.strip_prefix(b"---\n").is_some_and(|contents| { + contents.starts_with(crate::render::markdown::FRONTMATTER_HEADER.as_bytes()) + }) || (contents.starts_with(b"---\n") + && contents + .windows(header.len()) + .any(|window| window == header.as_bytes())))) } fn format(root: &Path, staged: &[StagedWrite]) -> Result<()> { - for target in Target::ALL { + for layout in LAYOUTS { let paths = staged .iter() - .filter(|file| file.output_target == target) + .filter(|file| std::ptr::eq(file.layout, layout)) .map(|file| file.temporary.clone()) .collect::>(); if !paths.is_empty() { - target.format(root, &paths)?; + format_layout(layout.formatter, root, &paths)?; } } Ok(()) } +fn format_layout(formatter: Formatter, root: &Path, paths: &[PathBuf]) -> Result<()> { + match formatter { + Formatter::None => Ok(()), + Formatter::Rust => format_rust(paths), + Formatter::Python => format_python(root, paths), + Formatter::C => format_c(paths), + Formatter::Cpp => format_cpp(paths), + } +} + pub(super) fn format_rust(paths: &[PathBuf]) -> Result<()> { run("rustfmt", &["--edition", "2024"], paths, None) } @@ -203,10 +308,13 @@ fn remove_temporary(staged: &[StagedWrite]) { #[cfg(test)] mod tests { - use std::{fs, path::Path}; + use std::{collections::BTreeMap, fs, path::Path}; use super::*; - use crate::targets::{GeneratedFile, documentation::HEADER}; + use crate::{ + output::{GeneratedFile, Output, REFERENCE_PYTHON}, + render::markdown::HEADER, + }; fn temporary_root(label: &str) -> PathBuf { std::env::temp_dir().join(format!( @@ -233,8 +341,8 @@ mod tests { ) .expect("write C page"); - let output = TargetOutput::new( - Target::PythonDocumentation, + let output = Output::new( + &REFERENCE_PYTHON, vec![GeneratedFile::new( "index.md".into(), format!("{HEADER}# Python\n"), @@ -255,7 +363,35 @@ mod tests { format!("---\ntitle: \"ptfkit.test\"\n---\n\n{HEADER}::: ptfkit.test\n").as_bytes(), HEADER, )); + assert!(is_generated( + format!( + "---\n{}title: ptfkit.test\n---\n", + crate::render::markdown::FRONTMATTER_HEADER + ) + .as_bytes(), + HEADER, + )); assert!(!is_generated(b"# Handwritten page\n", HEADER)); assert!(Path::new("index.md").is_relative()); } + + #[test] + fn generated_output_drift_reports_added_removed_and_modified_files() { + let before = BTreeMap::from([ + (PathBuf::from("generated/modified"), b"before".to_vec()), + (PathBuf::from("generated/removed"), b"removed".to_vec()), + ]); + let after = BTreeMap::from([ + (PathBuf::from("generated/added"), b"added".to_vec()), + (PathBuf::from("generated/modified"), b"after".to_vec()), + ]); + + assert_eq!( + drift_report(&before, &after), + Some( + "added:\n generated/added\nremoved:\n generated/removed\nmodified:\n generated/modified" + .to_owned() + ) + ); + } } diff --git a/codegen/src/render/c.rs b/codegen/src/render/c.rs new file mode 100644 index 0000000..1939645 --- /dev/null +++ b/codegen/src/render/c.rs @@ -0,0 +1,333 @@ +use std::fmt; + +use crate::semantic::{BinaryOp, Expr, MathFunction, Reference, UnaryOp, Variable}; + +#[derive(Clone, Copy)] +pub(crate) enum Dialect { + C, + Cpp, +} + +pub(crate) fn expression<'a>( + value: &'a Expr, + inputs: &'a [String], + variables: &'a [Variable], + dialect: Dialect, +) -> impl fmt::Display + 'a { + Expression { + expression: value, + inputs, + variables, + dialect, + } +} + +pub(crate) fn float_literal(lexeme: &str) -> String { + if lexeme.contains(['.', 'e', 'E']) { + lexeme.to_owned() + } else { + format!("{lexeme}.0") + } +} + +pub(crate) fn test_float_literal(value: f64) -> String { + float_literal(&value.to_string()) +} + +pub(crate) fn requires_math(expression: &Expr) -> bool { + match expression { + Expr::Number(_) | Expr::Reference(_) => false, + Expr::Unary { operand, .. } => requires_math(operand), + Expr::Binary { op, left, right } => { + matches!(op, BinaryOp::Power) || requires_math(left) || requires_math(right) + } + Expr::Call { .. } => true, + } +} + +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] +enum Precedence { + Sum, + Product, + Unary, + Primary, +} + +struct Expression<'a> { + expression: &'a Expr, + inputs: &'a [String], + variables: &'a [Variable], + dialect: Dialect, +} + +impl fmt::Display for Expression<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.write_expression(formatter, self.expression, None) + } +} + +impl Expression<'_> { + fn write_expression( + &self, + formatter: &mut fmt::Formatter<'_>, + expression: &Expr, + parent: Option<(Precedence, bool)>, + ) -> fmt::Result { + if let Expr::Unary { + op: UnaryOp::Plus, + operand, + } = expression + { + return self.write_expression(formatter, operand, parent); + } + + let precedence = precedence(expression); + let parenthesize = parent.is_some_and(|(parent, is_right)| { + precedence < parent || (is_right && precedence == parent) + }); + if parenthesize { + write!(formatter, "(")?; + } + + match expression { + Expr::Number(number) => write!(formatter, "{}", float_literal(&number.lexeme))?, + Expr::Reference(Reference::Input(index)) => { + write!(formatter, "{}", self.inputs[*index])? + } + Expr::Reference(Reference::Variable(index)) => { + write!(formatter, "{}", self.variables[*index].name)?; + } + Expr::Unary { op, operand } => match op { + UnaryOp::Plus => unreachable!("unary plus is handled before parenthesizing"), + UnaryOp::Minus => { + write!(formatter, "-")?; + self.write_expression(formatter, operand, Some((Precedence::Unary, true)))?; + } + }, + Expr::Binary { op, left, right } => match op { + BinaryOp::Add => self.write_binary(formatter, left, right, Precedence::Sum, "+")?, + BinaryOp::Subtract => { + self.write_binary(formatter, left, right, Precedence::Sum, "-")? + } + BinaryOp::Multiply => { + self.write_binary(formatter, left, right, Precedence::Product, "*")? + } + BinaryOp::Divide => { + self.write_binary(formatter, left, right, Precedence::Product, "/")? + } + BinaryOp::Power => { + write!(formatter, "{}(", self.math_name("pow"))?; + self.write_expression(formatter, left, None)?; + write!(formatter, ", ")?; + self.write_expression(formatter, right, None)?; + write!(formatter, ")")?; + } + }, + Expr::Call { function, args } => { + write!(formatter, "{}(", self.function_name(*function))?; + for (index, argument) in args.iter().enumerate() { + if index > 0 { + write!(formatter, ", ")?; + } + self.write_expression(formatter, argument, None)?; + } + write!(formatter, ")")?; + } + } + + if parenthesize { + write!(formatter, ")")?; + } + Ok(()) + } + + fn write_binary( + &self, + formatter: &mut fmt::Formatter<'_>, + left: &Expr, + right: &Expr, + precedence: Precedence, + operator: &str, + ) -> fmt::Result { + self.write_expression(formatter, left, Some((precedence, false)))?; + write!(formatter, " {operator} ")?; + self.write_expression(formatter, right, Some((precedence, true))) + } + + fn function_name(&self, function: MathFunction) -> &'static str { + match function { + MathFunction::Sqrt => self.math_name("sqrt"), + MathFunction::Exp => self.math_name("exp"), + MathFunction::Ln => self.math_name("log"), + MathFunction::Log10 => self.math_name("log10"), + MathFunction::Abs if matches!(self.dialect, Dialect::Cpp) => "std::abs", + MathFunction::Abs => self.math_name("fabs"), + MathFunction::Min => self.math_name("fmin"), + MathFunction::Max => self.math_name("fmax"), + } + } + + fn math_name(&self, name: &'static str) -> &'static str { + match self.dialect { + Dialect::C => name, + Dialect::Cpp => match name { + "sqrt" => "std::sqrt", + "exp" => "std::exp", + "log" => "std::log", + "log10" => "std::log10", + "fabs" => "std::fabs", + "fmin" => "std::fmin", + "fmax" => "std::fmax", + "pow" => "std::pow", + _ => unreachable!("unsupported C/C++ math function"), + }, + } + } +} + +fn precedence(expression: &Expr) -> Precedence { + match expression { + Expr::Binary { + op: BinaryOp::Add | BinaryOp::Subtract, + .. + } => Precedence::Sum, + Expr::Binary { + op: BinaryOp::Multiply | BinaryOp::Divide, + .. + } => Precedence::Product, + Expr::Unary { + op: UnaryOp::Minus, .. + } => Precedence::Unary, + Expr::Unary { + op: UnaryOp::Plus, + operand, + } => precedence(operand), + Expr::Number(_) | Expr::Reference(_) | Expr::Binary { .. } | Expr::Call { .. } => { + Precedence::Primary + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::semantic::{BinaryOp, Reference}; + + fn inputs() -> Vec { + vec!["x".into(), "y".into(), "z".into()] + } + + fn input(index: usize) -> Expr { + Expr::Reference(Reference::Input(index)) + } + + #[test] + fn preserves_precedence_for_c_and_cpp() { + let expression = Expr::Binary { + op: BinaryOp::Subtract, + left: Box::new(input(0)), + right: Box::new(Expr::Binary { + op: BinaryOp::Subtract, + left: Box::new(input(1)), + right: Box::new(input(2)), + }), + }; + + assert_eq!( + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), + "x - (y - z)" + ); + assert_eq!( + super::expression(&expression, &inputs(), &[], Dialect::Cpp).to_string(), + "x - (y - z)" + ); + } + + #[test] + fn preserves_parentheses_in_nested_unary_and_binary_expressions() { + let expression = Expr::Binary { + op: BinaryOp::Divide, + left: Box::new(Expr::Unary { + op: UnaryOp::Minus, + operand: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + }), + right: Box::new(Expr::Binary { + op: BinaryOp::Multiply, + left: Box::new(input(2)), + right: Box::new(Expr::Binary { + op: BinaryOp::Subtract, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + }), + }; + + assert_eq!( + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), + "-(x + y) / (z * (x - y))" + ); + } + + #[test] + fn unary_plus_preserves_the_operand_precedence() { + let expression = Expr::Binary { + op: BinaryOp::Multiply, + left: Box::new(Expr::Unary { + op: UnaryOp::Plus, + operand: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + }), + right: Box::new(input(2)), + }; + + assert_eq!( + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), + "(x + y) * z" + ); + } + + #[test] + fn renders_power_and_calls_with_dialect_local_math_names() { + let expression = Expr::Call { + function: MathFunction::Min, + args: vec![ + Expr::Binary { + op: BinaryOp::Power, + left: Box::new(Expr::Binary { + op: BinaryOp::Add, + left: Box::new(input(0)), + right: Box::new(input(1)), + }), + right: Box::new(Expr::Unary { + op: UnaryOp::Minus, + operand: Box::new(input(2)), + }), + }, + Expr::Call { + function: MathFunction::Sqrt, + args: vec![Expr::Binary { + op: BinaryOp::Multiply, + left: Box::new(input(0)), + right: Box::new(input(1)), + }], + }, + ], + }; + + assert_eq!( + super::expression(&expression, &inputs(), &[], Dialect::C).to_string(), + "fmin(pow(x + y, -z), sqrt(x * y))" + ); + assert_eq!( + super::expression(&expression, &inputs(), &[], Dialect::Cpp).to_string(), + "std::fmin(std::pow(x + y, -z), std::sqrt(x * y))" + ); + } +} diff --git a/codegen/src/render/markdown.rs b/codegen/src/render/markdown.rs new file mode 100644 index 0000000..7ad8a48 --- /dev/null +++ b/codegen/src/render/markdown.rs @@ -0,0 +1,56 @@ +//! Shared Markdown file and block rendering primitives. + +use std::path::PathBuf; + +use super::Writer; +use crate::output::GeneratedFile; + +pub(crate) const HEADER: &str = "\n\n"; +pub(crate) const FRONTMATTER_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n\n"; + +pub(crate) fn markdown_file( + path: impl Into, + render: impl FnOnce(&mut Writer), +) -> GeneratedFile { + let mut writer = Writer::new(); + render(&mut writer); + GeneratedFile::new(path.into(), markdown_contents(writer)) +} + +pub(crate) fn markdown_contents(writer: Writer) -> String { + format!("{}\n", writer.into_string().trim_end()) +} + +pub(crate) fn frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { + writer.line("---"); + render(writer); + writer.write("---\n\n"); +} + +pub(crate) fn generated_frontmatter(writer: &mut Writer, render: impl FnOnce(&mut Writer)) { + frontmatter(writer, |writer| { + writer.write(FRONTMATTER_HEADER); + render(writer); + }); +} + +pub(crate) fn code_block(writer: &mut Writer, language: &str, render: impl FnOnce(&mut Writer)) { + writer.line(format_args!("```{language}")); + render(writer); + writer.write("```\n\n"); +} + +pub(crate) fn admonition( + writer: &mut Writer, + kind: &str, + body: &str, + render_line: impl Fn(&str) -> String, +) { + writer.write(format_args!("!!! {kind}\n\n")); + writer.indented(|writer| { + for line in body.lines() { + writer.line(render_line(line)); + } + }); + writer.blank_line(); +} diff --git a/codegen/src/render/mod.rs b/codegen/src/render/mod.rs new file mode 100644 index 0000000..810e3c4 --- /dev/null +++ b/codegen/src/render/mod.rs @@ -0,0 +1,127 @@ +use std::fmt::{self, Display, Write as _}; + +pub(crate) mod c; +pub(crate) mod markdown; + +const DEFAULT_CAPACITY: usize = 16 * 1024; + +/// Renders a structured value into a [`Writer`]. +pub(crate) trait Render { + fn render(&self, writer: &mut Writer); +} + +/// A small, indentation-aware buffer for textual generators. +pub(crate) struct Writer { + contents: String, + indentation: usize, + indent: &'static str, + at_line_start: bool, +} + +impl Writer { + pub(crate) fn new() -> Self { + Self::with_capacity(DEFAULT_CAPACITY) + } + + /// Creates a writer with space reserved for at least `capacity` bytes. + pub(crate) fn with_capacity(capacity: usize) -> Self { + Self { + contents: String::with_capacity(capacity), + indentation: 0, + indent: " ", + at_line_start: true, + } + } + + #[allow(dead_code)] + pub(crate) fn with_indent(mut self, indent: &'static str) -> Self { + self.indent = indent; + self + } + + /// Writes one text fragment, adding indentation only at its start. + /// + /// Keep large static templates as single fragments outside an indented + /// block. Within [`Self::indented`], write each line separately with + /// [`Self::line`] or [`Self::blank_line`]; this method does not inspect + /// embedded newlines. + pub(crate) fn write(&mut self, value: impl Display) { + self.write_fmt(format_args!("{value}")) + .expect("writing to a String cannot fail"); + } + + pub(crate) fn line(&mut self, value: impl Display) { + self.write_fmt(format_args!("{value}\n")) + .expect("writing to a String cannot fail"); + } + + pub(crate) fn blank_line(&mut self) { + let res = if self.at_line_start { + self.write_char('\n') + } else { + self.write_str("\n\n") + }; + res.expect("writing to a String cannot fail"); + } + + /// Renders a nested block with one additional indentation level. + pub(crate) fn indented(&mut self, render: impl FnOnce(&mut Self)) { + self.indentation += 1; + render(self); + self.indentation -= 1; + } + + pub(crate) fn into_string(self) -> String { + self.contents + } +} + +impl fmt::Write for Writer { + fn write_str(&mut self, text: &str) -> fmt::Result { + debug_assert!( + self.indentation == 0 || !text.contains('\n') || text == "\n", + "write each indented line separately" + ); + if self.at_line_start && !text.is_empty() && !text.starts_with('\n') { + for _ in 0..self.indentation { + self.contents.push_str(self.indent); + } + } + self.contents.push_str(text); + self.at_line_start = text.ends_with('\n'); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{Render, Writer}; + + struct Paragraph(&'static str); + + impl Render for Paragraph { + fn render(&self, writer: &mut Writer) { + writer.line(self.0); + } + } + + #[test] + fn writes_display_values_and_blank_lines() { + let mut writer = Writer::new(); + writer.write("value: "); + writer.line(42); + writer.blank_line(); + writer.line("next"); + + assert_eq!(writer.into_string(), "value: 42\n\nnext\n"); + } + + #[test] + fn composes_render_values() { + let mut writer = Writer::new(); + Paragraph("first").render(&mut writer); + Paragraph("second").render(&mut writer); + + assert_eq!(writer.into_string(), "first\nsecond\n"); + } +} diff --git a/codegen/src/targets/c_documentation.rs b/codegen/src/targets/c_documentation.rs deleted file mode 100644 index 7604aec..0000000 --- a/codegen/src/targets/c_documentation.rs +++ /dev/null @@ -1,365 +0,0 @@ -use std::{collections::BTreeSet, path::PathBuf}; - -use anyhow::Result; - -use crate::model::{CompiledFunction, Output, Parameter}; - -use super::{ - GeneratedFile, - documentation::{FRONTMATTER_HEADER, HEADER}, - group_by_source, - native::c_result_name, -}; - -pub(super) fn render(functions: &[CompiledFunction]) -> Result> { - let sources = group_by_source(functions); - let mut files = vec![file("index.md", index(&sources))]; - files.push(header_file( - "headers/ptfkit.md", - "ptfkit", - umbrella(&sources), - )); - - let mut index_entries = Vec::new(); - for (slug, functions) in sources { - files.push(header_file( - format!("headers/{slug}.md"), - slug, - header(slug, &functions)?, - )); - for function in functions { - index_entries.push((slug, function)); - } - } - index_entries.sort_by_key(|(_, function)| natural_sort_key(&function.core.name)); - files.push(file("functions.md", functions_index(&index_entries))); - Ok(files) -} - -fn file(path: impl Into, contents: String) -> GeneratedFile { - GeneratedFile::new(path.into(), format!("{}\n", contents.trim_end())) -} - -fn header_file(path: impl Into, slug: &str, contents: String) -> GeneratedFile { - GeneratedFile::new( - path.into(), - format!("---\ntitle: \"{slug}.h\"\n---\n\n{}\n", contents.trim_end()), - ) -} - -fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C API reference\n---\n\n"); - text.push_str("# C API reference\n\n"); - text.push_str("ptfkit's C API is organized around installed headers.\n\n"); - text.push_str("## Headers\n\n"); - text.push_str( - "- [``](headers/ptfkit.md) — Aggregates every ptfkit source header.\n", - ); - for (slug, functions) in sources { - let summary = &functions[0].entry.spec.source.summary; - text.push_str(&format!( - "- [``](headers/{slug}.md) — {}\n", - escape_text(summary) - )); - } - text.push_str("\nSee the [function index](functions.md) for all public C functions.\n"); - text -} - -fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = String::from(HEADER); - text.push_str("# ``\n\n"); - text.push_str("```c\n#include \n```\n\n"); - text.push_str("This umbrella header aggregates every public ptfkit source header. Include an individual header when only one source is needed.\n\n"); - text.push_str("## Included headers\n\n"); - for (slug, functions) in sources { - text.push_str(&format!( - "- [``]({slug}.md) — {}\n", - escape_text(&functions[0].entry.spec.source.summary) - )); - } - text -} - -fn header(slug: &str, functions: &[&CompiledFunction]) -> Result { - let first = functions - .first() - .expect("compiled source contains at least one function"); - let source = &first.entry.spec.source; - let scope = &first.entry.spec.scope; - let mut text = String::from(HEADER); - text.push_str(&format!("# ``\n\n")); - text.push_str(&format!("```c\n#include \n```\n\n")); - text.push_str(&format!("{}\n\n", escape_text(&source.summary))); - text.push_str("## Source\n\n"); - text.push_str(&escape_text(&source.citation_apa)); - text.push_str("\n\n"); - if let Some(doi) = &source.doi { - text.push_str(&format!( - "[DOI: {}]({})\n\n", - escape_text(&doi.identifier), - doi.url - )); - } - if scope.territory.is_some() || scope.dataset.is_some() { - text.push_str("## Scope\n\n"); - if let Some(territory) = &scope.territory { - text.push_str(&format!("**Territory:** {}\n\n", escape_text(territory))); - } - if let Some(dataset) = &scope.dataset { - text.push_str(&format!("**Dataset:** {}\n\n", escape_text(dataset))); - } - } - text.push_str(&format!( - "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" - )); - - let mut structures = BTreeSet::new(); - for function in functions { - if let Output::Struct(_) = &function.core.output { - let spec = spec(function); - let name = spec - .result_class() - .expect("record output has a result class"); - let c_name = c_result_name(name); - if structures.insert(c_name.clone()) { - text.push_str(&structure(&c_name, spec.outputs.fields())); - } - } - } - text.push_str("## Functions\n\n"); - for function in functions { - text.push_str(&function_documentation(function)?); - } - Ok(text) -} - -fn structure(name: &str, fields: &[Parameter]) -> String { - let mut text = format!("## `{name}`\n\n"); - text.push_str("```c\n"); - text.push_str(&format!( - "typedef struct {{\n{} }} {name};\n", - fields - .iter() - .map(|field| format!(" double {};\n", field.name)) - .collect::() - )); - text.push_str("```\n\n| Field | Description |\n| --- | --- |\n"); - for field in fields { - text.push_str(&format!( - "| `{}` | {} |\n", - field.name, - parameter_details(field) - )); - } - text.push('\n'); - text -} - -fn function_documentation(function: &CompiledFunction) -> Result { - let spec = spec(function); - let anchor = function_anchor(&function.core.name); - let mut text = format!("### `{}` {{#{anchor}}}\n\n", function.core.name); - text.push_str(&format!("{}\n\n", escape_text(&spec.public_api.summary))); - text.push_str("```c\n"); - text.push_str(&signature(function)?); - text.push_str(";\n```\n\n"); - text.push_str("#### Parameters\n\n| Name | Direction | Description |\n| --- | --- | --- |\n"); - for parameter in &spec.inputs { - text.push_str(&format!( - "| `{}` | in | {} |\n", - parameter.name, - parameter_details(parameter) - )); - } - text.push_str("\n#### Returns\n\n"); - match &function.core.output { - Output::Scalar => text.push_str(&format!( - "{}\n\n", - parameter_details(&spec.outputs.fields()[0]) - )), - Output::Struct(_) => { - let name = spec - .result_class() - .expect("record output has a result class"); - text.push_str(&format!("A `{}` value.\n\n", c_result_name(name))); - } - } - for note in &spec.documentation.notes { - admonition(&mut text, "note", note); - } - for warning in &spec.documentation.warnings { - admonition(&mut text, "warning", warning); - } - Ok(text) -} - -fn functions_index(functions: &[(&str, &CompiledFunction)]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C function index\n---\n\n"); - text.push_str("# C function index\n\n| Function | Summary | Header |\n| --- | --- | --- |\n"); - for (slug, function) in functions { - text.push_str(&format!( - "| [`{0}`](headers/{1}.md#{2}) | {3} | [``](headers/{1}.md) |\n", - function.core.name, - slug, - function_anchor(&function.core.name), - escape_table(&spec(function).public_api.summary), - )); - } - text -} - -fn signature(function: &CompiledFunction) -> Result { - let spec = spec(function); - let result = match &function.core.output { - Output::Scalar => "double".to_owned(), - Output::Struct(_) => c_result_name( - spec.result_class() - .expect("record output has a result class"), - ), - }; - Ok(format!( - "static inline {result} {}({})", - function.core.name, - function - .core - .inputs - .iter() - .map(|input| format!("double {input}")) - .collect::>() - .join(", ") - )) -} - -fn spec(function: &CompiledFunction) -> &crate::model::Function { - &function.entry.spec.functions[function.function_index] -} - -fn function_anchor(name: &str) -> String { - format!("function-{name}") -} - -fn parameter_details(parameter: &Parameter) -> String { - format!( - "{} ({})", - escape_table(¶meter.description), - escape_table(¶meter.unit) - ) -} - -fn admonition(text: &mut String, kind: &str, body: &str) { - text.push_str(&format!("!!! {kind}\n\n")); - for line in body.lines() { - text.push_str(&format!(" {}\n", escape_text(line))); - } - text.push('\n'); -} - -fn escape_text(value: &str) -> String { - value.replace('\\', "\\\\").replace('`', "\\`") -} - -fn escape_table(value: &str) -> String { - escape_text(value).replace('\n', " ").replace('|', "\\|") -} - -fn natural_sort_key(value: &str) -> String { - super::py::natural_sort_key(value) -} - -#[cfg(test)] -mod tests { - use std::path::Path; - - use super::*; - - fn rendered_files() -> Vec { - let root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("codegen directory has a repository parent"); - let entries = crate::specs::load(root).expect("repository specifications load"); - let compiled = - super::super::compile::functions(entries).expect("repository specifications compile"); - render(&compiled).expect("C documentation renders") - } - - fn contents<'a>(files: &'a [GeneratedFile], path: &str) -> &'a str { - &files - .iter() - .find(|file| file.path == Path::new(path)) - .unwrap_or_else(|| panic!("missing generated file {path}")) - .contents - } - - #[test] - fn documents_headers_functions_and_record_fields_from_compiled_sources() { - let files = rendered_files(); - let index = contents(&files, "index.md"); - let rawls = contents(&files, "headers/rawls1982.md"); - let function_index = contents(&files, "functions.md"); - - assert!(index.starts_with( - "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: C API reference\n---\n" - )); - assert!(rawls.starts_with("---\ntitle: \"rawls1982.h\"\n---\n")); - assert!(index.contains("[``](headers/ptfkit.md)")); - assert!(rawls.contains("## `rawls1982_ptf_result`")); - assert!(rawls.contains("| `theta_4` | Volumetric water content at -4 kPa. (cm^3/cm^3) |")); - assert!(rawls.contains( - "static inline double calc_ptf_rawls1982_theta_1500(double clay, double organic_matter)" - )); - assert!(rawls.contains("static inline rawls1982_ptf_result calc_ptf_rawls1982_full_wrc")); - assert!(rawls.contains("{#function-calc_ptf_rawls1982_theta_1500}")); - assert!( - function_index.contains("headers/rawls1982.md#function-calc_ptf_rawls1982_theta_1500") - ); - assert!(rawls.contains("[PTF catalog page](../../../ptf-catalog/sources/rawls1982.md)")); - } - - #[test] - fn umbrella_lists_each_source_header_once_in_natural_order() { - let files = rendered_files(); - let umbrella = contents(&files, "headers/ptfkit.md"); - let index = contents(&files, "index.md"); - let source_headers = files - .iter() - .filter(|file| { - file.path.starts_with("headers") && file.path != Path::new("headers/ptfkit.md") - }) - .count(); - - assert_eq!(umbrella.matches("- [` Result { - Ok(rendered(expression, inputs, variables, dialect)?.text) -} - -pub(super) fn float_literal(lexeme: &str) -> String { - if lexeme.contains(['.', 'e', 'E']) { - lexeme.to_owned() - } else { - format!("{lexeme}.0") - } -} - -pub(super) fn test_float_literal(value: f64) -> String { - float_literal(&value.to_string()) -} - -pub(super) fn requires_math(expression: &Expr) -> bool { - match expression { - Expr::Number(_) | Expr::Reference(_) => false, - Expr::Unary { operand, .. } => requires_math(operand), - Expr::Binary { op, left, right } => { - matches!(op, BinaryOp::Power) || requires_math(left) || requires_math(right) - } - Expr::Call { .. } => true, - } -} - -#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] -enum Precedence { - Sum, - Product, - Unary, - Primary, -} - -struct RenderedExpression { - text: String, - precedence: Precedence, -} - -impl RenderedExpression { - fn binary_operand(&self, parent: Precedence, is_right: bool) -> String { - let needs_parentheses = self.precedence < parent || (is_right && self.precedence == parent); - if needs_parentheses { - format!("({})", self.text) - } else { - self.text.clone() - } - } - - fn unary_operand(&self) -> String { - if self.precedence <= Precedence::Unary { - format!("({})", self.text) - } else { - self.text.clone() - } - } -} - -fn rendered( - expression: &Expr, - inputs: &[String], - variables: &[Variable], - dialect: Dialect, -) -> Result { - Ok(match expression { - Expr::Number(number) => primary(float_literal(&number.lexeme)), - Expr::Reference(Reference::Input(index)) => primary(inputs[*index].clone()), - Expr::Reference(Reference::Variable(index)) => primary(variables[*index].name.clone()), - Expr::Unary { op, operand } => match op { - UnaryOp::Plus => rendered(operand, inputs, variables, dialect)?, - UnaryOp::Minus => { - let operand = rendered(operand, inputs, variables, dialect)?.unary_operand(); - RenderedExpression { - text: format!("-{operand}"), - precedence: Precedence::Unary, - } - } - }, - Expr::Binary { op, left, right } => { - let left = rendered(left, inputs, variables, dialect)?; - let right = rendered(right, inputs, variables, dialect)?; - match op { - BinaryOp::Add => binary(left, right, Precedence::Sum, "+"), - BinaryOp::Subtract => binary(left, right, Precedence::Sum, "-"), - BinaryOp::Multiply => binary(left, right, Precedence::Product, "*"), - BinaryOp::Divide => binary(left, right, Precedence::Product, "/"), - BinaryOp::Power => primary(format!( - "{}({}, {})", - math_name("pow", dialect), - left.text, - right.text - )), - } - } - Expr::Call { function, args } => { - let args = args - .iter() - .map(|arg| rendered(arg, inputs, variables, dialect)) - .collect::>>()?; - let name = match function { - MathFunction::Sqrt => "sqrt", - MathFunction::Exp => "exp", - MathFunction::Ln => "log", - MathFunction::Log10 => "log10", - MathFunction::Abs => "fabs", - MathFunction::Min => "fmin", - MathFunction::Max => "fmax", - }; - let name = match (dialect, function) { - (Dialect::Cpp, MathFunction::Abs) => "std::abs".to_owned(), - _ => math_name(name, dialect), - }; - primary(format!( - "{name}({})", - args.iter() - .map(|argument| argument.text.as_str()) - .collect::>() - .join(", ") - )) - } - }) -} - -fn math_name(name: &str, dialect: Dialect) -> String { - match dialect { - Dialect::C => name.to_owned(), - Dialect::Cpp => format!("std::{name}"), - } -} - -fn primary(text: String) -> RenderedExpression { - RenderedExpression { - text, - precedence: Precedence::Primary, - } -} - -fn binary( - left: RenderedExpression, - right: RenderedExpression, - precedence: Precedence, - operator: &str, -) -> RenderedExpression { - let left = left.binary_operand(precedence, false); - let right = right.binary_operand(precedence, true); - RenderedExpression { - text: format!("{left} {operator} {right}"), - precedence, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::semantic::{BinaryOp, Reference}; - - #[test] - fn preserves_precedence_for_c_and_cpp() { - let inputs = vec!["x".into(), "y".into(), "z".into()]; - let variables = Vec::new(); - let input = |index| Expr::Reference(Reference::Input(index)); - let expression = Expr::Binary { - op: BinaryOp::Subtract, - left: Box::new(input(0)), - right: Box::new(Expr::Binary { - op: BinaryOp::Subtract, - left: Box::new(input(1)), - right: Box::new(input(2)), - }), - }; - - assert_eq!( - render(&expression, &inputs, &variables, Dialect::C).unwrap(), - "x - (y - z)" - ); - assert_eq!( - render(&expression, &inputs, &variables, Dialect::Cpp).unwrap(), - "x - (y - z)" - ); - } - - #[test] - fn selects_cpp_math_namespace() { - let expression = Expr::Call { - function: MathFunction::Sqrt, - args: vec![Expr::Reference(Reference::Input(0))], - }; - let inputs = vec!["x".into()]; - - assert_eq!( - render(&expression, &inputs, &[], Dialect::C).unwrap(), - "sqrt(x)" - ); - assert_eq!( - render(&expression, &inputs, &[], Dialect::Cpp).unwrap(), - "std::sqrt(x)" - ); - } -} diff --git a/codegen/src/targets/catalog.rs b/codegen/src/targets/catalog.rs new file mode 100644 index 0000000..5bf3284 --- /dev/null +++ b/codegen/src/targets/catalog.rs @@ -0,0 +1,223 @@ +use crate::{ + documentation::{self as docs, FunctionDocument}, + model::{Entry, Parameter}, + output::GeneratedFile, + render::{Render, Writer, markdown}, +}; + +pub(super) fn render(entries: &[Entry]) -> Vec { + let mut files = vec![markdown::markdown_file("index.md", |writer| { + IndexPage { entries }.render(writer); + })]; + for entry in entries { + files.push(markdown::markdown_file( + format!("{}.md", entry.slug), + |writer| { + SourcePage { entry }.render(writer); + }, + )); + } + files +} + +struct IndexPage<'a> { + entries: &'a [Entry], +} + +impl Render for IndexPage<'_> { + fn render(&self, writer: &mut Writer) { + markdown::generated_frontmatter(writer, |writer| writer.line("title: PTF Sources")); + writer.write( + "# PTF Sources\n\ +\n\ +Each page describes the source, scope, inputs, outputs, status, and limitations of the functions defined by one specification.\n\ +\n\ +| Source | Territory | Functions |\n\ +| --- | --- | ---: |\n", + ); + for entry in self.entries { + let source = docs::for_source(&entry.spec.source, &entry.spec.scope); + let territory = source.territory.unwrap_or("\u{2014}"); + writer.line(format_args!( + "| [{}](./{}.md) | {} | {} |", + escape_table(source.summary), + entry.slug, + escape_table(territory), + entry.spec.functions.len(), + )); + } + } +} + +struct SourcePage<'a> { + entry: &'a Entry, +} + +impl Render for SourcePage<'_> { + fn render(&self, writer: &mut Writer) { + let spec = &self.entry.spec; + let source = docs::for_source(&spec.source, &spec.scope); + markdown::generated_frontmatter(writer, |writer| { + writer.line(format_args!("title: PTF source {}", self.entry.slug)); + writer.line(format_args!("nav-title: {}", self.entry.slug)); + }); + writer.write(format_args!( + "# {}\n\n## Source\n\n{}\n\n", + source.summary, source.reference.citation + )); + if let Some(doi) = source.reference.doi { + writer.write(format_args!("[DOI: {}]({})\n\n", doi.identifier, doi.url)); + } + if source.territory.is_some() || source.dataset.is_some() { + writer.write("## Scope\n\n"); + if let Some(territory) = source.territory { + writer.write(format_args!("**Territory:** {territory}\n\n")); + } + if let Some(dataset) = source.dataset { + writer.write(format_args!("**Dataset:** {dataset}\n\n")); + } + } + writer.write("## Functions\n\n"); + for function in &spec.functions { + FunctionSection { + document: docs::for_function(function), + name: &function.public_api.name, + status: &function.status, + } + .render(writer); + } + } +} + +struct FunctionSection<'a> { + document: FunctionDocument<'a>, + name: &'a str, + status: &'a str, +} + +impl Render for FunctionSection<'_> { + fn render(&self, writer: &mut Writer) { + writer.write(format_args!( + "### `{}`\n\n{}\n\n**Status:** `{}`\n\n**Prediction target:** {}\n\n", + self.name, self.document.summary, self.status, self.document.remarks.prediction_target, + )); + + if self.document.models.h_theta.is_some() || self.document.models.k_h.is_some() { + writer.write("**Models:** "); + if let Some(model) = self.document.models.h_theta { + writer.write(format_args!("$h(\\theta)$ \u{2014} {model}")); + if self.document.models.k_h.is_some() { + writer.write("; "); + } + } + if let Some(model) = self.document.models.k_h { + writer.write(format_args!("$k(h)$ \u{2014} {model}")); + } + writer.blank_line(); + } + + ParameterTable { + title: "Inputs", + parameters: self.document.parameters, + } + .render(writer); + ParameterTable { + title: "Outputs", + parameters: match self.document.returns { + docs::Returns::Scalar(field) => std::slice::from_ref(field), + docs::Returns::Record { fields, .. } => fields, + }, + } + .render(writer); + for note in self.document.notes { + Admonition { + kind: "note", + body: note, + } + .render(writer); + } + for warning in self.document.warnings { + Admonition { + kind: "warning", + body: warning, + } + .render(writer); + } + } +} + +struct ParameterTable<'a> { + title: &'a str, + parameters: &'a [Parameter], +} + +impl Render for ParameterTable<'_> { + fn render(&self, writer: &mut Writer) { + writer.write(format_args!( + "#### {}\n\n| Name | Unit | Domain | Description |\n| --- | --- | --- | --- |\n", + self.title + )); + for parameter in self.parameters { + writer.line(format_args!( + "| `{}` | {} | {} | {} |", + parameter.name, + escape_table(¶meter.unit), + parameter + .domain + .as_deref() + .map(escape_table) + .unwrap_or_else(|| "\u{2014}".into()), + escape_table(¶meter.description), + )); + } + writer.blank_line(); + } +} + +struct Admonition<'a> { + kind: &'a str, + body: &'a str, +} + +impl Render for Admonition<'_> { + fn render(&self, writer: &mut Writer) { + markdown::admonition(writer, self.kind, self.body, str::to_owned); + } +} + +fn escape_table(value: &str) -> String { + value.replace('\n', " ").replace('|', "\\|") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn renders_admonitions_with_writer_indentation() { + let mut writer = Writer::new(); + Admonition { + kind: "note", + body: "first\nsecond", + } + .render(&mut writer); + + assert_eq!( + writer.into_string(), + "!!! note\n\n first\n second\n\n" + ); + } + + #[test] + fn renders_generated_frontmatter_with_the_existing_spacing() { + let file = markdown::markdown_file("test.md", |writer| { + markdown::generated_frontmatter(writer, |writer| writer.line("title: Test")); + writer.line("# Test"); + }); + + assert_eq!( + file.contents, + "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: Test\n---\n\n# Test\n" + ); + } +} diff --git a/codegen/src/targets/cpp_documentation.rs b/codegen/src/targets/cpp_documentation.rs deleted file mode 100644 index b8c6a6b..0000000 --- a/codegen/src/targets/cpp_documentation.rs +++ /dev/null @@ -1,334 +0,0 @@ -use std::{collections::BTreeSet, path::PathBuf}; - -use anyhow::{Result, anyhow}; - -use crate::model::{CompiledFunction, Output, Parameter}; - -use super::{GeneratedFile, documentation::FRONTMATTER_HEADER, group_by_source}; - -pub(super) fn render(functions: &[CompiledFunction]) -> Result> { - let sources = group_by_source(functions); - let mut files = vec![file("index.md", index(&sources))]; - files.push(GeneratedFile::new( - "modules/ptfkit.md".into(), - umbrella(&sources), - )); - - let mut index_entries = Vec::new(); - for (slug, functions) in sources { - files.push(GeneratedFile::new( - format!("modules/{slug}.md").into(), - module(slug, &functions)?, - )); - for function in functions { - index_entries.push((slug, function)); - } - } - index_entries.sort_by_key(|(_, function)| natural_sort_key(&function.core.name)); - files.push(file("functions.md", functions_index(&index_entries))); - Ok(files) -} - -fn file(path: impl Into, contents: String) -> GeneratedFile { - GeneratedFile::new(path.into(), format!("{}\n", contents.trim_end())) -} - -fn index(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C++ API reference\n---\n\n"); - text.push_str("# C++ API reference\n\n"); - text.push_str("ptfkit's C++ API is organized around C++20 modules.\n\n"); - text.push_str("## Modules\n\n"); - text.push_str("- [`ptfkit`](modules/ptfkit.md) — Re-exports every ptfkit source module.\n"); - for (slug, functions) in sources { - text.push_str(&format!( - "- [`ptfkit.{slug}`](modules/{slug}.md) — {}\n", - escape_text(&functions[0].entry.spec.source.summary) - )); - } - text.push_str("\nSee the [function index](functions.md) for all public C++ functions.\n"); - text -} - -fn umbrella(sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>) -> String { - let mut text = - format!("---\n{FRONTMATTER_HEADER}title: C++ module ptfkit\nnav-title: ptfkit\n---\n\n"); - text.push_str("# `ptfkit`\n\n```cpp\nimport ptfkit;\n```\n\n"); - text.push_str("This umbrella module re-exports every public ptfkit source module. Import an individual module when only one source is needed.\n\n"); - text.push_str("## Re-exported modules\n\n"); - for (slug, functions) in sources { - text.push_str(&format!( - "- [`ptfkit.{slug}`]({slug}.md) — {}\n", - escape_text(&functions[0].entry.spec.source.summary) - )); - } - text -} - -fn module(slug: &str, functions: &[&CompiledFunction]) -> Result { - let first = functions - .first() - .expect("compiled source contains at least one function"); - let source = &first.entry.spec.source; - let scope = &first.entry.spec.scope; - let mut text = format!( - "---\n{FRONTMATTER_HEADER}title: C++ module ptfkit.{slug}\nnav-title: ptfkit.{slug}\n---\n\n" - ); - text.push_str(&format!( - "# `ptfkit.{slug}`\n\n```cpp\nimport ptfkit.{slug};\n```\n\n" - )); - text.push_str(&format!("**Exported namespace:** `ptfkit::{slug}`\n\n")); - text.push_str(&format!("{}\n\n", escape_text(&source.summary))); - text.push_str("## Source\n\n"); - text.push_str(&escape_text(&source.citation_apa)); - text.push_str("\n\n"); - if let Some(doi) = &source.doi { - text.push_str(&format!( - "[DOI: {}]({})\n\n", - escape_text(&doi.identifier), - doi.url - )); - } - if scope.territory.is_some() || scope.dataset.is_some() { - text.push_str("## Scope\n\n"); - if let Some(territory) = &scope.territory { - text.push_str(&format!("**Territory:** {}\n\n", escape_text(territory))); - } - if let Some(dataset) = &scope.dataset { - text.push_str(&format!("**Dataset:** {}\n\n", escape_text(dataset))); - } - } - text.push_str(&format!( - "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" - )); - - let mut structures = BTreeSet::new(); - for function in functions { - if let Output::Struct(_) = &function.core.output { - let result = result_class(function)?; - if structures.insert(result) { - text.push_str(&structure(result, spec(function).outputs.fields())); - } - } - } - text.push_str("## Functions\n\n"); - for function in functions { - text.push_str(&function_documentation(function)?); - } - text.pop(); - Ok(text) -} - -fn structure(name: &str, fields: &[Parameter]) -> String { - let mut text = format!("## `{name}`\n\n```cpp\nstruct {name} {{\n"); - for field in fields { - text.push_str(&format!(" double {};\n", field.name)); - } - text.push_str("};\n```\n\n| Field | Description |\n| --- | --- |\n"); - for field in fields { - text.push_str(&format!( - "| `{}` | {} |\n", - field.name, - parameter_details(field) - )); - } - text.push('\n'); - text -} - -fn function_documentation(function: &CompiledFunction) -> Result { - let spec = spec(function); - let anchor = function_anchor(&function.core.name); - let mut text = format!("### `{}` {{#{anchor}}}\n\n", function.core.name); - text.push_str(&format!("{}\n\n", escape_text(&spec.public_api.summary))); - text.push_str("```cpp\n"); - text.push_str(&signature(function)?); - text.push_str("\n```\n\n"); - text.push_str("#### Parameters\n\n| Name | Description |\n| --- | --- |\n"); - for parameter in &spec.inputs { - text.push_str(&format!( - "| `{}` | {} |\n", - parameter.name, - parameter_details(parameter) - )); - } - text.push_str("\n#### Returns\n\n"); - match &function.core.output { - Output::Scalar => text.push_str(&format!( - "{}\n\n", - parameter_details(&spec.outputs.fields()[0]) - )), - Output::Struct(_) => text.push_str(&format!("A `{}` value.\n\n", result_class(function)?)), - } - for note in &spec.documentation.notes { - admonition(&mut text, "note", note); - } - for warning in &spec.documentation.warnings { - admonition(&mut text, "warning", warning); - } - Ok(text) -} - -fn functions_index(functions: &[(&str, &CompiledFunction)]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: C++ function index\n---\n\n"); - text.push_str("# C++ function index\n\n| Function | Summary | Module |\n| --- | --- | --- |\n"); - for (slug, function) in functions { - let qualified = format!("ptfkit::{slug}::{}", function.core.name); - text.push_str(&format!( - "| [`{qualified}`](modules/{slug}.md#{}) | {} | [`ptfkit.{slug}`](modules/{slug}.md) |\n", - function_anchor(&function.core.name), - escape_table(&spec(function).public_api.summary), - )); - } - text -} - -fn signature(function: &CompiledFunction) -> Result { - let result = match &function.core.output { - Output::Scalar => "double".to_owned(), - Output::Struct(_) => result_class(function)?.to_owned(), - }; - Ok(format!( - "[[nodiscard]]\ninline {result} {}({})", - function.core.name, - function - .core - .inputs - .iter() - .map(|input| format!("double {input}")) - .collect::>() - .join(", ") - )) -} - -fn spec(function: &CompiledFunction) -> &crate::model::Function { - &function.entry.spec.functions[function.function_index] -} - -fn result_class(function: &CompiledFunction) -> Result<&str> { - spec(function) - .result_class() - .ok_or_else(|| anyhow!("record output has no result class")) -} - -fn function_anchor(name: &str) -> String { - format!("function-{name}") -} - -fn parameter_details(parameter: &Parameter) -> String { - format!( - "{} ({})", - escape_table(¶meter.description), - escape_table(¶meter.unit) - ) -} - -fn admonition(text: &mut String, kind: &str, body: &str) { - text.push_str(&format!("!!! {kind}\n\n")); - for line in body.lines() { - text.push_str(&format!(" {}\n", escape_text(line))); - } - text.push('\n'); -} - -fn escape_text(value: &str) -> String { - value.replace('\\', "\\\\").replace('`', "\\`") -} - -fn escape_table(value: &str) -> String { - escape_text(value).replace('\n', " ").replace('|', "\\|") -} - -fn natural_sort_key(value: &str) -> String { - super::py::natural_sort_key(value) -} - -#[cfg(test)] -mod tests { - use std::path::Path; - - use super::*; - - fn rendered_files() -> Vec { - let root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("codegen directory has a repository parent"); - let entries = crate::specs::load(root).expect("repository specifications load"); - let compiled = - super::super::compile::functions(entries).expect("repository specifications compile"); - render(&compiled).expect("C++ documentation renders") - } - - fn contents<'a>(files: &'a [GeneratedFile], path: &str) -> &'a str { - &files - .iter() - .find(|file| file.path == Path::new(path)) - .unwrap_or_else(|| panic!("missing generated file {path}")) - .contents - } - - #[test] - fn documents_modules_functions_and_record_fields_from_compiled_sources() { - let files = rendered_files(); - let index = contents(&files, "index.md"); - let rawls = contents(&files, "modules/rawls1982.md"); - let function_index = contents(&files, "functions.md"); - - assert!(index.starts_with( - "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: C++ API reference\n---\n" - )); - assert!(rawls.starts_with( - "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: C++ module ptfkit.rawls1982\nnav-title: ptfkit.rawls1982\n---\n" - )); - assert!(index.contains("[`ptfkit`](modules/ptfkit.md)")); - assert!(rawls.contains("import ptfkit.rawls1982;")); - assert!(rawls.contains("**Exported namespace:** `ptfkit::rawls1982`")); - assert!(rawls.contains("## `Rawls1982PTFResult`")); - assert!(rawls.contains("| `theta_4` | Volumetric water content at -4 kPa. (cm^3/cm^3) |")); - assert!(rawls.contains("[[nodiscard]]\ninline double calc_ptf_rawls1982_theta_1500")); - assert!(rawls.contains("inline Rawls1982PTFResult calc_ptf_rawls1982_full_wrc")); - assert!(rawls.contains("{#function-calc_ptf_rawls1982_theta_1500}")); - assert!(function_index.contains( - "[`ptfkit::rawls1982::calc_ptf_rawls1982_theta_1500`](modules/rawls1982.md#function-calc_ptf_rawls1982_theta_1500)" - )); - assert!(rawls.contains("[PTF catalog page](../../../ptf-catalog/sources/rawls1982.md)")); - } - - #[test] - fn umbrella_lists_each_source_module_once_in_natural_order() { - let files = rendered_files(); - let umbrella = contents(&files, "modules/ptfkit.md"); - let index = contents(&files, "index.md"); - let source_modules = files - .iter() - .filter(|file| { - file.path.starts_with("modules") && file.path != Path::new("modules/ptfkit.md") - }) - .count(); - - assert_eq!(umbrella.matches("- [`ptfkit.").count(), source_modules); - assert!(index.find("[`ptfkit`](").unwrap() < index.find("ptfkit.ahuja1984").unwrap()); - assert!( - umbrella.find("ptfkit.ahuja1984").unwrap() - < umbrella.find("ptfkit.aimrun2009").unwrap() - ); - } - - #[test] - fn escapes_markdown_sensitive_text() { - assert_eq!(escape_text("a `name` \\ value"), "a \\`name\\` \\\\ value"); - assert_eq!(escape_table("a|b\nc"), "a\\|b c"); - } - - #[test] - fn function_anchors_are_stable() { - assert_eq!( - function_anchor("calc_ptf_rawls1982"), - "function-calc_ptf_rawls1982" - ); - } - - #[test] - fn natural_ordering_handles_numeric_suffixes() { - assert!(natural_sort_key("calc_2") < natural_sort_key("calc_10")); - } -} diff --git a/codegen/src/targets/documentation.rs b/codegen/src/targets/documentation.rs deleted file mode 100644 index 8e98234..0000000 --- a/codegen/src/targets/documentation.rs +++ /dev/null @@ -1,151 +0,0 @@ -use std::path::PathBuf; - -use crate::model::{Entry, Function, Parameter}; - -pub(super) const HEADER: &str = "\n\n"; -pub(super) const FRONTMATTER_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n\n"; - -pub(super) fn parameter_details(parameter: &Parameter) -> String { - format!("{} ({})", parameter.description, parameter.unit) -} - -pub(super) fn parameter_documentation(parameter: &Parameter) -> String { - format!("{}: {}", parameter.name, parameter_details(parameter)) -} - -pub(super) fn render(entries: &[Entry]) -> Vec { - let mut files = vec![super::GeneratedFile::new( - PathBuf::from("index.md"), - format!("{}\n", render_index(entries).trim_end()), - )]; - for entry in entries { - files.push(super::GeneratedFile::new( - PathBuf::from(format!("{}.md", entry.slug)), - format!("{}\n", render_source(entry).trim_end()), - )); - } - files -} - -fn render_index(entries: &[Entry]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: PTF Sources\n---\n\n"); - text.push_str("# PTF Sources\n\n"); - text.push_str( - "Each page describes the source, scope, inputs, outputs, status, and limitations of the \ - functions defined by one specification.\n\n", - ); - text.push_str("| Source | Territory | Functions |\n| --- | --- | ---: |\n"); - for entry in entries { - let spec = &entry.spec; - let territory = spec.scope.territory.as_deref().unwrap_or("—"); - text.push_str(&format!( - "| [{}](./{}.md) | {} | {} |\n", - escape_table(&spec.source.summary), - entry.slug, - escape_table(territory), - spec.functions.len(), - )); - } - text -} - -fn render_source(entry: &Entry) -> String { - let spec = &entry.spec; - let mut text = format!( - "---\n{FRONTMATTER_HEADER}title: PTF source {slug}\nnav-title: {slug}\n---\n\n", - slug = entry.slug - ); - text.push_str(&format!("# {}\n\n", spec.source.summary)); - text.push_str("## Source\n\n"); - text.push_str(&spec.source.citation_apa); - text.push_str("\n\n"); - if let Some(doi) = &spec.source.doi { - text.push_str(&format!("[DOI: {}]({})\n\n", doi.identifier, doi.url)); - } - if spec.scope.territory.is_some() || spec.scope.dataset.is_some() { - text.push_str("## Scope\n\n"); - if let Some(territory) = &spec.scope.territory { - text.push_str(&format!("**Territory:** {territory}\n\n")); - } - if let Some(dataset) = &spec.scope.dataset { - text.push_str(&format!("**Dataset:** {dataset}\n\n")); - } - } - text.push_str("## Functions\n\n"); - for function in &spec.functions { - render_function(&mut text, function); - } - text -} - -fn render_function(text: &mut String, function: &Function) { - text.push_str(&format!("### `{}`\n\n", function.public_api.name)); - text.push_str(&function.public_api.summary); - text.push_str("\n\n"); - text.push_str(&format!("**Status:** `{}`\n\n", function.status)); - text.push_str(&format!( - "**Prediction target:** {}\n\n", - function.scope.prediction_target - )); - - let models = [ - function - .scope - .models - .h_theta - .as_ref() - .map(|model| format!("$h(\\theta)$ — {model}")), - function - .scope - .models - .k_h - .as_ref() - .map(|model| format!("$k(h)$ — {model}")), - ] - .into_iter() - .flatten() - .collect::>(); - if !models.is_empty() { - text.push_str(&format!("**Models:** {}\n\n", models.join("; "))); - } - - render_parameters(text, "Inputs", &function.inputs); - render_parameters(text, "Outputs", function.outputs.fields()); - for note in &function.documentation.notes { - render_admonition(text, "note", note); - } - for warning in &function.documentation.warnings { - render_admonition(text, "warning", warning); - } -} - -fn render_parameters(text: &mut String, title: &str, parameters: &[Parameter]) { - text.push_str(&format!("#### {title}\n\n")); - text.push_str("| Name | Unit | Domain | Description |\n| --- | --- | --- | --- |\n"); - for parameter in parameters { - text.push_str(&format!( - "| `{}` | {} | {} | {} |\n", - parameter.name, - escape_table(¶meter.unit), - parameter - .domain - .as_deref() - .map(escape_table) - .unwrap_or_else(|| "—".into()), - escape_table(¶meter.description), - )); - } - text.push('\n'); -} - -fn render_admonition(text: &mut String, kind: &str, body: &str) { - text.push_str(&format!("!!! {kind}\n\n")); - for line in body.lines() { - text.push_str(&format!(" {line}\n")); - } - text.push('\n'); -} - -fn escape_table(value: &str) -> String { - value.replace('\n', " ").replace('|', "\\|") -} diff --git a/codegen/src/targets/mod.rs b/codegen/src/targets/mod.rs index 37d6a5f..b0d1afc 100644 --- a/codegen/src/targets/mod.rs +++ b/codegen/src/targets/mod.rs @@ -1,22 +1,20 @@ -mod c_documentation; -mod c_expression; -mod compile; -mod cpp_documentation; -mod documentation; +//! Render concrete generated products from compiled specifications. + +mod catalog; mod native; -mod py; -mod python_documentation; -mod rs; -mod write; +mod python; +mod reference; +mod rust; -use std::{ - collections::BTreeMap, - path::{Path, PathBuf}, -}; +use std::{collections::BTreeMap, path::Path}; use anyhow::Result; -use crate::model::{CompiledFunction, Entry, PythonGeneration}; +use crate::{ + compile, + model::{CompiledFunction, Entry}, + output::{self, Output}, +}; pub(super) fn group_by_source( functions: &[CompiledFunction], @@ -31,223 +29,39 @@ pub(super) fn group_by_source( sources } -#[derive(Clone, Copy, PartialEq, Eq)] -pub(super) enum Target { - Documentation, - CDocumentation, - CppDocumentation, - PythonDocumentation, - Rust, - PythonExtension, - PythonWrapper, - PythonTest, - NativeC, - NativeCppModule, - NativeCppCmake, - NativeCTest, - NativeCppTest, -} - -impl Target { - pub(super) const ALL: [Self; 13] = [ - Self::Documentation, - Self::CDocumentation, - Self::CppDocumentation, - Self::PythonDocumentation, - Self::Rust, - Self::PythonExtension, - Self::PythonWrapper, - Self::PythonTest, - Self::NativeC, - Self::NativeCppModule, - Self::NativeCppCmake, - Self::NativeCTest, - Self::NativeCppTest, - ]; - - fn output_path(self, root: &Path, relative: &Path) -> PathBuf { - match self { - Self::Documentation => root.join("docs/src/ptf-catalog/sources").join(relative), - Self::CDocumentation => root.join("docs/src/reference/c").join(relative), - Self::CppDocumentation => root.join("docs/src/reference/cpp").join(relative), - Self::PythonDocumentation => root.join("docs/src/reference/python").join(relative), - Self::Rust => root.join("targets/ptfkit-rs/src").join(relative), - Self::PythonExtension => root.join("targets/ptfkit-py").join(relative), - Self::PythonWrapper => root.join("targets/ptfkit-py/src").join(relative), - Self::PythonTest => root.join("targets/ptfkit-py").join(relative), - Self::NativeC => root.join("targets/ptfkit-native/include").join(relative), - Self::NativeCppModule => root.join("targets/ptfkit-native/cpp").join(relative), - Self::NativeCppCmake => root.join("targets/ptfkit-native/cmake").join(relative), - Self::NativeCTest => root.join("targets/ptfkit-native/tests/c").join(relative), - Self::NativeCppTest => root.join("targets/ptfkit-native/tests/cpp").join(relative), - } - } - - fn cleanup_directory(self, root: &Path) -> PathBuf { - match self { - Self::Documentation => root.join("docs/src/ptf-catalog/sources"), - Self::CDocumentation => root.join("docs/src/reference/c"), - Self::CppDocumentation => root.join("docs/src/reference/cpp"), - Self::PythonDocumentation => root.join("docs/src/reference/python"), - Self::Rust => root.join("targets/ptfkit-rs/src"), - Self::PythonExtension | Self::PythonWrapper => { - root.join("targets/ptfkit-py/src/ptfkit") - } - Self::PythonTest => root.join("targets/ptfkit-py/tests"), - Self::NativeC => root.join("targets/ptfkit-native/include"), - Self::NativeCppModule => root.join("targets/ptfkit-native/cpp"), - Self::NativeCppCmake => root.join("targets/ptfkit-native/cmake"), - Self::NativeCTest => root.join("targets/ptfkit-native/tests/c"), - Self::NativeCppTest => root.join("targets/ptfkit-native/tests/cpp"), - } - } - - fn generated_header(self) -> &'static str { - match self { - Self::Documentation => documentation::HEADER, - Self::CDocumentation => documentation::HEADER, - Self::CppDocumentation => documentation::HEADER, - Self::PythonDocumentation => documentation::HEADER, - Self::Rust => rs::HEADER, - Self::PythonExtension => py::C_HEADER, - Self::PythonWrapper => py::WRAPPER_HEADER, - Self::PythonTest => py::WRAPPER_HEADER, - Self::NativeC | Self::NativeCppModule | Self::NativeCTest | Self::NativeCppTest => { - native::HEADER - } - Self::NativeCppCmake => native::CMAKE_HEADER, - } - } - - fn is_clang_formatted(self) -> bool { - matches!( - self, - Self::PythonExtension - | Self::NativeC - | Self::NativeCppModule - | Self::NativeCTest - | Self::NativeCppTest - ) - } - - fn format(self, root: &Path, paths: &[PathBuf]) -> Result<()> { - match self { - Self::Documentation - | Self::CDocumentation - | Self::CppDocumentation - | Self::PythonDocumentation => Ok(()), - Self::Rust => write::format_rust(paths), - Self::PythonExtension | Self::NativeCTest => write::format_c(paths), - Self::PythonWrapper => write::format_python(root, paths), - Self::PythonTest => write::format_python(root, paths), - Self::NativeC | Self::NativeCppModule | Self::NativeCppTest => write::format_cpp(paths), - Self::NativeCppCmake => Ok(()), - } - } -} - -pub(super) struct TargetOutput { - pub(super) target: Target, - pub(super) files: Vec, -} - -impl TargetOutput { - fn new(target: Target, files: Vec) -> Self { - Self { target, files } - } -} - -pub(super) struct GeneratedFile { - path: PathBuf, - contents: String, -} - -impl GeneratedFile { - fn new(path: PathBuf, contents: String) -> Self { - Self { path, contents } - } -} - pub(crate) fn run(root: &Path, entries: Vec) -> Result<()> { - let documentation = documentation::render(&entries); - let python_documentation = python_documentation::render(&entries); + let catalog = catalog::render(&entries); + let reference_python = reference::python::render(&entries); let compiled = compile::functions(entries)?; - let rust = rs::render(&compiled)? - .into_iter() - .map(|(path, contents)| GeneratedFile::new(path, contents)) - .collect::>(); - let py = py::render(&compiled)?; + let reference_c = reference::c::render(&compiled)?; + let reference_cpp = reference::cpp::render(&compiled)?; + let rust = rust::render(&compiled)?; + let python = python::render(&compiled)?; let native = native::render(&compiled)?; - let c_documentation = c_documentation::render(&compiled)?; - let cpp_documentation = cpp_documentation::render(&compiled)?; - let c = py - .c_sources - .into_iter() - .map(|(path, contents)| GeneratedFile::new(path.into(), contents)) - .collect::>(); - let mut wrappers = py - .wrappers - .into_iter() - .filter_map(|(module, mode, contents)| { - (mode == PythonGeneration::Generated).then_some(GeneratedFile::new( - PathBuf::from(module.replace('.', "/")).with_extension("py"), - contents, - )) - }) - .collect::>(); - wrappers.push(GeneratedFile::new("ptfkit/_ptfkit.pyi".into(), py.stub)); - let tests = py - .tests - .into_iter() - .filter_map(|(slug, mode, contents)| { - (mode == PythonGeneration::Generated).then_some(GeneratedFile::new( - format!("tests/test_{slug}.py").into(), - contents, - )) - }) - .collect::>(); - write::commit( + + output::commit( root, &[ - TargetOutput::new(Target::Documentation, documentation), - TargetOutput::new(Target::CDocumentation, c_documentation), - TargetOutput::new(Target::CppDocumentation, cpp_documentation), - TargetOutput::new(Target::PythonDocumentation, python_documentation), - TargetOutput::new(Target::Rust, rust), - TargetOutput::new(Target::PythonExtension, c), - TargetOutput::new(Target::PythonWrapper, wrappers), - TargetOutput::new(Target::PythonTest, tests), - TargetOutput::new(Target::NativeC, native.c_headers), - TargetOutput::new(Target::NativeCppModule, native.cpp_modules), - TargetOutput::new(Target::NativeCppCmake, native.cpp_cmake), - TargetOutput::new(Target::NativeCTest, native.c_tests), - TargetOutput::new(Target::NativeCppTest, native.cpp_tests), + Output::new(&output::CATALOG, catalog), + Output::new(&output::REFERENCE_C, reference_c), + Output::new(&output::REFERENCE_CPP, reference_cpp), + Output::new(&output::REFERENCE_PYTHON, reference_python), + Output::new(&output::RUST, rust), + Output::new(&output::PYTHON_EXTENSION, python.extension), + Output::new(&output::PYTHON_WRAPPER, python.wrappers), + Output::new(&output::PYTHON_TEST, python.tests), + Output::new(&output::NATIVE_C, native.c_headers), + Output::new(&output::NATIVE_CPP_MODULE, native.cpp_modules), + Output::new(&output::NATIVE_CPP_CMAKE, native.cpp_cmake), + Output::new(&output::NATIVE_C_TEST, native.c_tests), + Output::new(&output::NATIVE_CPP_TEST, native.cpp_tests), ], ) } -#[cfg(test)] -mod tests { - use std::path::Path; - - use super::Target; - - #[test] - fn documentation_targets_use_structured_mkdocs_roots() { - let root = Path::new("repository"); - let relative = Path::new("index.md"); - - for (target, directory) in [ - (Target::Documentation, "docs/src/ptf-catalog/sources"), - (Target::CDocumentation, "docs/src/reference/c"), - (Target::CppDocumentation, "docs/src/reference/cpp"), - (Target::PythonDocumentation, "docs/src/reference/python"), - ] { - assert_eq!( - target.output_path(root, relative), - root.join(directory).join(relative) - ); - assert_eq!(target.cleanup_directory(root), root.join(directory)); - } - } +/// Regenerate every target and fail when that changes a codegen-owned file. +pub(crate) fn check_generated(root: &Path, entries: Vec) -> Result<()> { + let before = output::snapshot_generated(root)?; + run(root, entries)?; + output::assert_unchanged(root, before) } diff --git a/codegen/src/targets/native.rs b/codegen/src/targets/native.rs index dbf9007..bb59a0c 100644 --- a/codegen/src/targets/native.rs +++ b/codegen/src/targets/native.rs @@ -3,22 +3,27 @@ use std::{collections::BTreeSet, path::PathBuf}; use anyhow::Result; use convert_case::{Boundary, Case, Casing}; -use crate::model::{CompiledFunction, Function, Output, Parameter, Scope, Source}; +use crate::{ + documentation::{self as docs, FunctionDocument, SourceDocument}, + model::{CompiledFunction, Function, Output, Parameter, Scope, Source}, + render::{Render, Writer}, +}; -use super::{ - c_expression::{self, Dialect}, - documentation, group_by_source, +use crate::{ + output::GeneratedFile, + render::c::{self, Dialect}, + targets::group_by_source, }; pub(super) const HEADER: &str = "/* @generated by ptfkit-codegen; DO NOT EDIT. */\n"; pub(super) const CMAKE_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n"; pub(super) struct OutputFiles { - pub(super) c_headers: Vec, - pub(super) cpp_modules: Vec, - pub(super) cpp_cmake: Vec, - pub(super) c_tests: Vec, - pub(super) cpp_tests: Vec, + pub(super) c_headers: Vec, + pub(super) cpp_modules: Vec, + pub(super) cpp_cmake: Vec, + pub(super) c_tests: Vec, + pub(super) cpp_tests: Vec, } pub(super) fn render(functions: &[CompiledFunction]) -> Result { @@ -26,57 +31,54 @@ pub(super) fn render(functions: &[CompiledFunction]) -> Result { let mut cpp_modules = Vec::new(); let mut c_tests = Vec::new(); let mut cpp_tests = Vec::new(); - let mut c_includes = String::new(); + let mut umbrella = Writer::new(); + umbrella.write(format_args!( + "{HEADER}\n\n#ifndef PTFKIT_PTFKIT_H\n#define PTFKIT_PTFKIT_H\n\n" + )); + + let mut root_module = Writer::new(); + root_module.write(format_args!("{HEADER}\n\nexport module ptfkit;\n\n")); + let mut module_paths = vec!["cpp/ptfkit.cppm".to_owned()]; for (slug, functions) in group_by_source(functions) { - c_includes.push_str(&format!("#include \n")); + umbrella.line(format_args!("#include ")); c_headers.push(file( format!("ptfkit/{slug}.h"), c_header(slug, &functions)?, )); module_paths.push(format!("cpp/{slug}.cppm")); + root_module.line(format_args!("export import ptfkit.{slug};")); cpp_modules.push(file(format!("{slug}.cppm"), cpp_module(slug, &functions)?)); c_tests.push(file(format!("{slug}.c"), c_test(slug, &functions)?)); cpp_tests.push(file(format!("{slug}.cpp"), cpp_test(slug, &functions)?)); } - c_headers.push(file( - "ptfkit/ptfkit.h", - format!( - "{HEADER}\n#ifndef PTFKIT_PTFKIT_H\n#define PTFKIT_PTFKIT_H\n\n{c_includes}\n#endif\n" - ), - )); - let exports = module_paths - .iter() - .skip(1) - .map(|path| path.trim_start_matches("cpp/").trim_end_matches(".cppm")) - .map(|slug| format!("export import ptfkit.{slug};\n")) - .collect::(); - cpp_modules.push(file( - "ptfkit.cppm", - format!("{HEADER}\nexport module ptfkit;\n\n{exports}"), - )); - let module_paths = module_paths - .iter() - .map(|path| format!(" \"${{CMAKE_CURRENT_LIST_DIR}}/../{path}\"")) - .collect::>() - .join("\n"); + umbrella.write("\n\n#endif\n"); + c_headers.push(file("ptfkit/ptfkit.h", umbrella.into_string())); + + cpp_modules.push(file("ptfkit.cppm", root_module.into_string())); + + let mut cmake = Writer::new(); + cmake.write(format_args!("{CMAKE_HEADER}\nset(PTFKIT_CPP_MODULES\n")); + cmake.indented(|writer| { + for path in &module_paths { + writer.line(format_args!("\"${{CMAKE_CURRENT_LIST_DIR}}/../{path}\"")); + } + }); + cmake.line(")"); Ok(OutputFiles { c_headers, cpp_modules, - cpp_cmake: vec![file( - "ptfkitModules.cmake", - format!("{CMAKE_HEADER}set(PTFKIT_CPP_MODULES\n{module_paths}\n)\n"), - )], + cpp_cmake: vec![file("ptfkitModules.cmake", cmake.into_string())], c_tests, cpp_tests, }) } -fn file(path: impl Into, contents: String) -> super::GeneratedFile { - super::GeneratedFile::new(path.into(), contents) +fn file(path: impl Into, contents: String) -> GeneratedFile { + GeneratedFile::new(path.into(), contents) } -pub(super) fn c_result_name(schema: &str) -> String { +pub(crate) fn c_result_name(schema: &str) -> String { schema .remove_boundaries(&[Boundary::LowerDigit]) .to_case(Case::Snake) @@ -84,17 +86,17 @@ pub(super) fn c_result_name(schema: &str) -> String { fn c_header(slug: &str, functions: &[&CompiledFunction]) -> Result { let guard = format!("PTFKIT_{}_H", slug.to_ascii_uppercase()); - let mut body = String::new(); + let mut writer = Writer::new(); + writer.write(format_args!( + "{HEADER}\n\n#ifndef {guard}\n#define {guard}\n\n" + )); if requires_math(functions) { - body.push_str("#include \n"); + writer.write("#include \n\n"); } let first = functions .first() .expect("generated source contains at least one function"); - body.push_str(&format!( - "\n{}", - source_comment(&first.entry.spec.source, &first.entry.spec.scope) - )); + source_comment(&first.entry.spec.source, &first.entry.spec.scope).render(&mut writer); let mut schemas = BTreeSet::new(); for function in functions { let spec = &function.entry.spec.functions[function.function_index]; @@ -103,277 +105,358 @@ fn c_header(slug: &str, functions: &[&CompiledFunction]) -> Result { .result_class() .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; if schemas.insert(schema) { - body.push_str(&format!( - "\ntypedef struct {{\n{} }} {};\n", - fields - .iter() - .map(|field| { - let parameter = spec - .outputs - .fields() - .iter() - .find(|parameter| parameter.name == *field) - .expect("core output field matches specification"); - format!(" {}\n double {field};\n", field_comment(parameter)) - }) - .collect::(), - c_result_name(schema) - )); + writer.blank_line(); + render_struct( + &mut writer, + fields, + spec, + &c_result_name(schema), + NativeDialect::C, + ); } } } for function in functions { - body.push_str(&format!("\n{}", c_function(function)?)); + writer.blank_line(); + NativeFunction::c(function)?.render(&mut writer); } - Ok(format!( - "{HEADER}\n#ifndef {guard}\n#define {guard}\n\n{body}\n#endif\n" - )) + writer.write("\n\n#endif\n"); + Ok(writer.into_string()) } fn cpp_module(slug: &str, functions: &[&CompiledFunction]) -> Result { - let mut body = String::new(); + let mut writer = Writer::new(); + writer.write(format_args!("{HEADER}\n\n")); if requires_math(functions) { - body.push_str("module;\n#include \n\n"); + writer.write("module;\n#include \n\n"); } - body.push_str(&format!("export module ptfkit.{slug};\n\n")); + writer.write(format_args!("export module ptfkit.{slug};\n\n")); let first = functions .first() .expect("generated source contains at least one function"); - body.push_str(&source_comment( - &first.entry.spec.source, - &first.entry.spec.scope, - )); - body.push('\n'); - body.push_str(&format!("export namespace ptfkit::{slug} {{\n")); - let mut schemas = BTreeSet::new(); - for function in functions { - let spec = &function.entry.spec.functions[function.function_index]; - if let Output::Struct(fields) = &function.core.output { - let result = spec - .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; - if schemas.insert(result) { - body.push_str(&format!( - "\nstruct {} {{\n{} }};\n", - result, - fields - .iter() - .map(|field| { - let parameter = spec - .outputs - .fields() - .iter() - .find(|parameter| parameter.name == *field) - .expect("core output field matches specification"); - format!(" {}\n double {field};\n", field_comment(parameter)) - }) - .collect::() - )); + source_comment(&first.entry.spec.source, &first.entry.spec.scope).render(&mut writer); + writer.blank_line(); + let native_functions = functions + .iter() + .map(|function| NativeFunction::cpp(function)) + .collect::>>()?; + writer.line(format_args!("export namespace ptfkit::{slug} {{")); + writer.indented(|writer| { + let mut schemas = BTreeSet::new(); + for function in functions { + let spec = &function.entry.spec.functions[function.function_index]; + if let Output::Struct(fields) = &function.core.output { + let result = spec + .result_class() + .expect("record output has a result class"); + if schemas.insert(result) { + writer.blank_line(); + render_struct(writer, fields, spec, result, NativeDialect::Cpp); + } } } - } - for function in functions { - body.push_str(&format!("\n{}", cpp_function(function)?)); - } - body.push_str(&format!("\n}} // namespace ptfkit::{slug}\n")); - Ok(format!("{HEADER}\n{body}")) + for function in &native_functions { + writer.blank_line(); + function.render(writer); + } + }); + writer.write(format_args!("\n\n}} // namespace ptfkit::{slug}\n")); + Ok(writer.into_string()) } -fn c_function(function: &CompiledFunction) -> Result { - render_function(function, false) +#[derive(Clone, Copy)] +enum NativeDialect { + C, + Cpp, } -fn cpp_function(function: &CompiledFunction) -> Result { - render_function(function, true) + +struct NativeFunction<'a> { + function: &'a CompiledFunction, + result: String, + dialect: NativeDialect, + terminal: bool, } -fn render_function(function: &CompiledFunction, cpp: bool) -> Result { - let spec = &function.entry.spec.functions[function.function_index]; - let result = match function.core.output { - Output::Scalar => "double".to_owned(), - Output::Struct(_) if cpp => spec - .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))? - .to_owned(), - Output::Struct(_) => c_result_name( - spec.result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?, - ), - }; - let prefix = if cpp { - "[[nodiscard]]\ninline" - } else { - "static inline" - }; - let inputs = function - .core - .inputs - .iter() - .map(|input| format!("double {input}")) - .collect::>() - .join(", "); - let mut variables = String::new(); - let output_name = &spec.outputs.fields()[0].name; - let terminal = matches!(function.core.output, Output::Scalar) - && function - .ir - .variables - .last() - .is_some_and(|v| v.name == *output_name); - for variable in function - .ir - .variables - .iter() - .take(function.ir.variables.len() - usize::from(terminal)) - { - variables.push_str(&format!( - " const double {} = {};\n", - variable.name, - expression(&variable.expression, function, cpp)? - )); +impl<'a> NativeFunction<'a> { + fn c(function: &'a CompiledFunction) -> Result { + Self::new(function, NativeDialect::C) + } + + fn cpp(function: &'a CompiledFunction) -> Result { + Self::new(function, NativeDialect::Cpp) } - let returned = match &function.core.output { - Output::Scalar if terminal => expression( - &function + + fn new(function: &'a CompiledFunction, dialect: NativeDialect) -> Result { + let spec = &function.entry.spec.functions[function.function_index]; + let result = match function.core.output { + Output::Scalar => "double".to_owned(), + Output::Struct(_) if matches!(dialect, NativeDialect::Cpp) => spec + .result_class() + .ok_or_else(|| anyhow::anyhow!("record output has no result class"))? + .to_owned(), + Output::Struct(_) => c_result_name( + spec.result_class() + .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?, + ), + }; + let output_name = &spec.outputs.fields()[0].name; + let terminal = matches!(function.core.output, Output::Scalar) + && function .ir .variables .last() - .expect("terminal variable") - .expression, + .is_some_and(|variable| variable.name == *output_name); + Ok(Self { function, - cpp, - )?, - Output::Scalar => output_name.clone(), - Output::Struct(fields) if cpp => format!("{result}{{{}}}", fields.join(", ")), - Output::Struct(fields) => format!("{result}{{{}}}", fields.join(", ")), - }; - let return_statement = match &function.core.output { - Output::Struct(fields) if !cpp => format!( - "#ifdef __cplusplus\n return {result}{{{}}};\n#else\n return ({result}) {{\n{} }};\n#endif", - fields.join(", "), - fields + result, + dialect, + terminal, + }) + } +} + +impl Render for NativeFunction<'_> { + fn render(&self, writer: &mut Writer) { + let spec = &self.function.entry.spec.functions[self.function.function_index]; + function_comment(spec).render(writer); + if matches!(self.dialect, NativeDialect::Cpp) { + writer.line("[[nodiscard]]"); + writer.write("inline "); + } else { + writer.write("static inline "); + } + writer.write(&self.result); + writer.write(" "); + writer.write(&self.function.core.name); + writer.write("("); + for (index, input) in self.function.core.inputs.iter().enumerate() { + if index > 0 { + writer.write(", "); + } + writer.write("double "); + writer.write(input); + } + writer.line(") {"); + writer.indented(|writer| { + for variable in self + .function + .ir + .variables .iter() - .map(|field| format!(" .{field} = {field},\n")) - .collect::() - ), - _ => format!("return {returned};"), - }; - Ok(format!( - "{}{prefix} {result} {}({inputs}) {{\n{variables} {return_statement}\n}}\n", - function_comment(spec), - function.core.name - )) + .take(self.function.ir.variables.len() - usize::from(self.terminal)) + { + writer.write(format_args!("const double {} = ", variable.name)); + writer.write(c::expression( + &variable.expression, + &self.function.core.inputs, + &self.function.ir.variables, + self.expression_dialect(), + )); + writer.line(";"); + } + self.render_return(writer); + }); + writer.line("}"); + } +} + +impl NativeFunction<'_> { + fn expression_dialect(&self) -> Dialect { + match self.dialect { + NativeDialect::C => Dialect::C, + NativeDialect::Cpp => Dialect::Cpp, + } + } + + fn render_return(&self, writer: &mut Writer) { + let output_name = &self.function.entry.spec.functions[self.function.function_index] + .outputs + .fields()[0] + .name; + match &self.function.core.output { + Output::Scalar if self.terminal => { + writer.write("return "); + writer.write(c::expression( + &self + .function + .ir + .variables + .last() + .expect("terminal variable") + .expression, + &self.function.core.inputs, + &self.function.ir.variables, + self.expression_dialect(), + )); + writer.line(";"); + } + Output::Scalar => writer.line(format_args!("return {output_name};")), + Output::Struct(fields) if matches!(self.dialect, NativeDialect::C) => { + writer.line("#ifdef __cplusplus"); + writer.write(format_args!("return {}{{", self.result)); + render_values(writer, fields); + writer.line("};"); + writer.line("#else"); + writer.line(format_args!("return ({}) {{", self.result)); + writer.indented(|writer| { + for field in fields { + writer.line(format_args!(".{field} = {field},")); + } + }); + writer.line("};"); + writer.line("#endif"); + } + Output::Struct(fields) => { + writer.write(format_args!("return {}{{", self.result)); + render_values(writer, fields); + writer.line("};"); + } + } + } +} + +fn render_values(writer: &mut Writer, values: &[String]) { + for (index, value) in values.iter().enumerate() { + if index > 0 { + writer.write(", "); + } + writer.write(value); + } } -fn source_comment(source: &Source, scope: &Scope) -> String { +fn render_struct( + writer: &mut Writer, + fields: &[String], + spec: &Function, + name: &str, + dialect: NativeDialect, +) { + match dialect { + NativeDialect::C => writer.line("typedef struct {"), + NativeDialect::Cpp => writer.line(format_args!("struct {name} {{")), + } + writer.indented(|writer| { + for field in fields { + let parameter = spec + .outputs + .fields() + .iter() + .find(|parameter| parameter.name == *field) + .expect("core output field matches specification"); + field_comment(parameter).render(writer); + writer.line(format_args!("double {field};")); + } + }); + match dialect { + NativeDialect::C => writer.line(format_args!("}} {name};")), + NativeDialect::Cpp => writer.line("};"), + } +} + +fn source_comment(source: &Source, scope: &Scope) -> Comment { + source_comment_from_document(docs::for_source(source, scope)) +} + +fn source_comment_from_document(document: SourceDocument<'_>) -> Comment { let mut lines = vec![ - format!("@brief {}", source.summary), + format!("@brief {}", document.summary), String::new(), "@details Source publication:".to_owned(), - source.citation_apa.clone(), + document.reference.citation.into(), ]; - if let Some(doi) = &source.doi { + if let Some(doi) = document.reference.doi { lines.push(format!("@see {} DOI: {}", doi.url, doi.identifier)); } - if let Some(territory) = &scope.territory { + if let Some(territory) = document.territory { lines.extend([ String::new(), "@remark Geographic scope:".to_owned(), - territory.clone(), + territory.into(), ]); } - if let Some(dataset) = &scope.dataset { + if let Some(dataset) = document.dataset { lines.extend([ String::new(), "@remark Calibration dataset:".to_owned(), - dataset.clone(), + dataset.into(), ]); } - comment(lines) + Comment(lines) +} + +fn function_comment(function: &Function) -> Comment { + function_comment_from_document(docs::for_function(function)) } -fn function_comment(function: &Function) -> String { - let mut lines = vec![format!("@brief {}", function.public_api.summary)]; - lines.extend(function.inputs.iter().map(|parameter| { +fn function_comment_from_document(document: FunctionDocument<'_>) -> Comment { + let mut lines = vec![format!("@brief {}", document.summary)]; + lines.extend(document.parameters.iter().map(|parameter| { format!( "@param {} {}", parameter.name, - documentation::parameter_details(parameter) + docs::parameter_details(parameter) ) })); - let outputs = function.outputs.fields(); + let outputs = match document.returns { + docs::Returns::Scalar(field) => std::slice::from_ref(field), + docs::Returns::Record { fields, .. } => fields, + }; if outputs.len() == 1 { - lines.push(format!( - "@return {}", - documentation::parameter_details(&outputs[0]) - )); + lines.push(format!("@return {}", docs::parameter_details(&outputs[0]))); } else { lines.push("@return A result with the following fields:".to_owned()); lines.extend(outputs.iter().map(|parameter| { format!( "- `{}` — {}", parameter.name, - documentation::parameter_details(parameter) + docs::parameter_details(parameter) ) })); } - if let Some(territory) = &function.scope.territory { + if let Some(territory) = document.territory { lines.extend([ String::new(), "@remark Geographic scope:".to_owned(), - territory.clone(), + territory.into(), ]); } lines.extend([ String::new(), "@details Prediction target:".to_owned(), - function.scope.prediction_target.clone(), + document.remarks.prediction_target.into(), ]); + lines.extend(document.notes.iter().map(|note| format!("@note {note}"))); lines.extend( - function - .documentation - .notes - .iter() - .map(|note| format!("@note {note}")), - ); - lines.extend( - function - .documentation + document .warnings .iter() .map(|warning| format!("@warning {warning}")), ); - comment(lines) + Comment(lines) } -fn field_comment(parameter: &Parameter) -> String { - comment([format!( +fn field_comment(parameter: &Parameter) -> Comment { + Comment(vec![format!( "@brief {}", - documentation::parameter_details(parameter) + docs::parameter_details(parameter) )]) - .trim_end() - .to_owned() } -fn comment(lines: impl IntoIterator) -> String { - let lines = lines - .into_iter() - .flat_map(|line| wrap_comment_line(&line)) - .map(|line| line.replace("*/", "* /")) - .collect::>(); - format!( - "/**\n{} */\n", - lines - .into_iter() - .map(|line| { - if line.is_empty() { - " *\n".to_owned() - } else { - format!(" * {line}\n") - } - }) - .collect::() - ) +struct Comment(Vec); + +impl Render for Comment { + fn render(&self, writer: &mut Writer) { + writer.line("/**"); + for line in self.0.iter().flat_map(|line| wrap_comment_line(line)) { + if line.is_empty() { + writer.line(" *"); + } else { + writer.line(format_args!(" * {}", line.replace("*/", "* /"))); + } + } + writer.line(" */"); + } } fn wrap_comment_line(line: &str) -> Vec { @@ -397,26 +480,13 @@ fn wrap_comment_line(line: &str) -> Vec { lines } -fn expression( - expression: &crate::semantic::Expr, - function: &CompiledFunction, - cpp: bool, -) -> Result { - c_expression::render( - expression, - &function.core.inputs, - &function.ir.variables, - if cpp { Dialect::Cpp } else { Dialect::C }, - ) -} - fn requires_math(functions: &[&CompiledFunction]) -> bool { functions.iter().any(|function| { function .ir .variables .iter() - .any(|variable| c_expression::requires_math(&variable.expression)) + .any(|variable| c::requires_math(&variable.expression)) }) } @@ -428,102 +498,114 @@ fn cpp_test(slug: &str, functions: &[&CompiledFunction]) -> Result { } fn c_compatibility_test(slug: &str, functions: &[&CompiledFunction]) -> Result { - let includes = ""; - let mut tests = String::new(); - for function in functions { - let spec = &function.entry.spec.functions[function.function_index]; - for case in &function.golden_tests { - let values = case - .inputs - .iter() - .map(|value| c_expression::test_float_literal(*value)) - .collect::>(); - let call = format!("{}({})", function.core.name, values.join(", ")); - let result_declaration = if matches!(function.core.output, Output::Scalar) { - "const double" - } else { - "const ptfkit_result_placeholder" - }; - let result_declaration = if result_declaration == "const ptfkit_result_placeholder" { - let name = spec - .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; - format!("const {}", c_result_name(name)) - } else { - result_declaration.to_owned() - }; - tests.push_str(&format!( - " {{\n {result_declaration} result = {call};\n" - )); - for (field, expected) in spec.outputs.fields().iter().zip(&case.expected) { - let actual = if matches!(function.core.output, Output::Scalar) { - "result".to_owned() - } else { - format!("result.{}", field.name) - }; - tests.push_str(&format!( - " assert_close_enough({actual}, {}, {}, {});\n", - c_expression::test_float_literal(*expected), - c_expression::test_float_literal(case.atol), - c_expression::test_float_literal(case.rtol), - )); + let mut writer = Writer::new(); + writer.write(format_args!( + "{HEADER}\n\n#include \n#include \"close_enough.h\"\n\n" + )); + writer.line("int main() {"); + writer.indented(|writer| { + for function in functions { + let spec = &function.entry.spec.functions[function.function_index]; + for case in &function.golden_tests { + writer.line("{"); + writer.indented(|writer| { + if matches!(function.core.output, Output::Scalar) { + writer.write("const double"); + } else { + let name = spec + .result_class() + .expect("record output has a result class"); + writer.write(format_args!("const {}", c_result_name(name))); + } + writer.write(" result = "); + writer.write(&function.core.name); + writer.write("("); + render_literals(writer, &case.inputs); + writer.line(");"); + for (field, expected) in spec.outputs.fields().iter().zip(&case.expected) { + writer.write("assert_close_enough("); + if matches!(function.core.output, Output::Scalar) { + writer.write("result"); + } else { + writer.write(format_args!("result.{}", field.name)); + } + writer.write(", "); + writer.write(c::test_float_literal(*expected)); + writer.write(", "); + writer.write(c::test_float_literal(case.atol)); + writer.write(", "); + writer.write(c::test_float_literal(case.rtol)); + writer.line(");"); + } + }); + writer.line("}"); } - tests.push_str(" }\n"); } - } - Ok(format!( - "{HEADER}\n#include \n#include \"close_enough.h\"\n{includes}\nint main() {{\n{tests} return 0;\n}}\n" - )) + writer.line("return 0;"); + }); + writer.line("}"); + Ok(writer.into_string()) } fn module_test(slug: &str, functions: &[&CompiledFunction]) -> Result { let type_traits = functions .iter() .any(|function| matches!(function.core.output, Output::Struct(_))); - let includes = if type_traits { - "#include \n" + let mut writer = Writer::new(); + writer.write(format_args!("{HEADER}\n\n#ifdef IMPORT_UMBRELLA\nimport ptfkit;\n#else\nimport ptfkit.{slug};\n#endif\n\n#include \"close_enough.h\"")); + if type_traits { + writer.write("\n#include \n\n"); } else { - "" - }; - let mut tests = String::new(); + writer.blank_line(); + } + writer.line("int main() {"); + writer.indented(|writer| { for function in functions { let spec = &function.entry.spec.functions[function.function_index]; for case in &function.golden_tests { - let values = case - .inputs - .iter() - .map(|value| c_expression::test_float_literal(*value)) - .collect::>(); - tests.push_str(&format!( - " {{\n const auto result = ptfkit::{slug}::{}({});\n", - function.core.name, - values.join(", ") - )); + writer.line("{"); + writer.indented(|writer| { + writer.write(format_args!("const auto result = ptfkit::{slug}::{}(", function.core.name)); + render_literals(writer, &case.inputs); + writer.line(");"); if matches!(function.core.output, Output::Struct(_)) { let result = spec .result_class() - .ok_or_else(|| anyhow::anyhow!("record output has no result class"))?; - tests.push_str(&format!(" static_assert(std::is_same_v, ptfkit::{slug}::{result}>);\n")); + .expect("record output has a result class"); + writer.line(format_args!("static_assert(std::is_same_v, ptfkit::{slug}::{result}>);")); } for (field, expected) in spec.outputs.fields().iter().zip(&case.expected) { - let actual = if matches!(function.core.output, Output::Scalar) { - "result".to_owned() + writer.write("assert_close_enough("); + if matches!(function.core.output, Output::Scalar) { + writer.write("result"); } else { - format!("result.{}", field.name) - }; - tests.push_str(&format!( - " assert_close_enough({actual}, {}, {}, {});\n", - c_expression::test_float_literal(*expected), - c_expression::test_float_literal(case.atol), - c_expression::test_float_literal(case.rtol), - )); + writer.write(format_args!("result.{}", field.name)); + } + writer.write(", "); + writer.write(c::test_float_literal(*expected)); + writer.write(", "); + writer.write(c::test_float_literal(case.atol)); + writer.write(", "); + writer.write(c::test_float_literal(case.rtol)); + writer.line(");"); } - tests.push_str(" }\n"); + }); + writer.line("}"); + } + } + writer.line("return 0;"); + }); + writer.line("}"); + Ok(writer.into_string()) +} + +fn render_literals(writer: &mut Writer, values: &[f64]) { + for (index, value) in values.iter().enumerate() { + if index > 0 { + writer.write(", "); } + writer.write(c::test_float_literal(*value)); } - Ok(format!( - "{HEADER}\n#ifdef IMPORT_UMBRELLA\nimport ptfkit;\n#else\nimport ptfkit.{slug};\n#endif\n\n#include \"close_enough.h\"\n{includes}\nint main() {{\n{tests} return 0;\n}}\n" - )) } #[cfg(test)] @@ -552,9 +634,9 @@ mod tests { args: vec![value], }; - assert!(!c_expression::requires_math(&arithmetic)); - assert!(c_expression::requires_math(&power)); - assert!(c_expression::requires_math(&logarithm)); + assert!(!c::requires_math(&arithmetic)); + assert!(c::requires_math(&power)); + assert!(c::requires_math(&logarithm)); } #[test] diff --git a/codegen/src/targets/py/c.rs b/codegen/src/targets/py/c.rs deleted file mode 100644 index f68ed59..0000000 --- a/codegen/src/targets/py/c.rs +++ /dev/null @@ -1,122 +0,0 @@ -use anyhow::Result; - -use crate::model::{CompiledFunction, Output}; - -use super::{ - super::{ - c_expression::{self, Dialect}, - group_by_source, - }, - C_HEADER, -}; - -pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { - let mut includes = String::new(); - let mut registers = Vec::new(); - let mut writes = Vec::new(); - for (slug, functions) in group_by_source(functions) { - let register = format!("ptfkit_register_{slug}"); - registers.push(register.clone()); - let mut definitions = String::new(); - let mut calls = String::new(); - for function in functions { - definitions.push_str(&ufunc(function)?); - calls.push_str(&format!( - " if (ptfkit_add_ufunc(module, \"{name}\", {name}_functions, {name}_types, {nin}, {nout}) < 0) return -1;\n", - name = function.core.name, - nin = function.core.inputs.len(), - nout = output_count(&function.core.output), - )); - } - includes.push_str(&format!("#include \"{slug}.c\"\n")); - writes.push(( - format!("src/ptfkit/{slug}.c"), - format!( - "{C_HEADER}#include \"ufunc.h\"\n\n{definitions}int {register}(PyObject *module) {{\n{calls} return 0;\n}}\n" - ), - )); - } - let calls = registers - .iter() - .map(|register| { - format!(" if ({register}(module) < 0) {{ Py_DECREF(module); return NULL; }}\n") - }) - .collect::(); - writes.push(( - "src/ptfkit/ptfkit.c".into(), - format!( - "{C_HEADER}#define PY_SSIZE_T_CLEAN\n#define PY_ARRAY_UNIQUE_SYMBOL PTFKIT_ARRAY_API\n#include \n#include \n#include \n\n{includes}\nstatic struct PyModuleDef module_def = {{ PyModuleDef_HEAD_INIT, \"_ptfkit\", NULL, -1, NULL }};\n\nPyMODINIT_FUNC PyInit__ptfkit(void) {{\n PyObject *module = PyModule_Create(&module_def);\n if (module == NULL) return NULL;\n import_array();\n import_ufunc();\n{calls} return module;\n}}\n" - ), - )); - Ok(writes) -} - -fn ufunc(function: &CompiledFunction) -> Result { - let name = &function.core.name; - let inputs = &function.core.inputs; - let mut locals = String::new(); - for (index, input) in inputs.iter().enumerate() { - locals.push_str(&format!( - " const double {input} = *(const double *)args[{index}];\n" - )); - } - for variable in &function.ir.variables { - locals.push_str(&format!( - " const double {} = {};\n", - variable.name, - c_expression::render( - &variable.expression, - inputs, - &function.ir.variables, - Dialect::C, - )? - )); - } - let values = match &function.core.output { - Output::Scalar => vec![ - function.entry.spec.functions[function.function_index] - .outputs - .fields()[0] - .name - .clone(), - ], - Output::Struct(fields) => fields.clone(), - }; - let writes = values - .iter() - .enumerate() - .map(|(index, value)| { - format!( - " *(double *)args[{}] = {value};\n", - inputs.len() + index - ) - }) - .collect::(); - let types = std::iter::repeat_n("NPY_DOUBLE", inputs.len() + values.len()) - .collect::>() - .join(", "); - Ok(format!( - "static void {name}_loop(char **args, const npy_intp *dimensions, const npy_intp *steps, void *data) {{\n npy_intp index;\n for (index = 0; index < dimensions[0]; index++) {{\n{locals}{writes} for (int arg = 0; arg < {}; arg++) args[arg] += steps[arg];\n }}\n}}\nstatic PyUFuncGenericFunction {name}_functions[] = {{ {name}_loop }};\nstatic char {name}_types[] = {{ {types} }};\n\n", - inputs.len() + values.len() - )) -} - -fn output_count(output: &Output) -> usize { - match output { - Output::Scalar => 1, - Output::Struct(fields) => fields.len(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn entry_source_initializes_the_private_module() { - let rendered = render(&[]).unwrap(); - let entry = rendered.last().unwrap().1.as_str(); - assert!(entry.contains("PyInit__ptfkit")); - assert!(!entry.contains("#include \"ufunc.h\"")); - } -} diff --git a/codegen/src/targets/py/test.rs b/codegen/src/targets/py/test.rs deleted file mode 100644 index 0e2d9dc..0000000 --- a/codegen/src/targets/py/test.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::collections::BTreeMap; - -use crate::model::{CompiledFunction, Function, Outputs, PythonGeneration}; - -use super::{WRAPPER_HEADER, natural_sort_key}; - -pub(super) fn render(functions: &[CompiledFunction]) -> Vec<(String, PythonGeneration, String)> { - let mut modules: BTreeMap> = BTreeMap::new(); - for function in functions { - modules - .entry(function.entry.slug.clone()) - .or_default() - .push(function); - } - - modules - .into_iter() - .map(|(slug, functions)| { - let mode = functions[0].entry.spec.generation.public_python; - let source = if mode == PythonGeneration::Generated { - module_source(&slug, &functions) - } else { - String::new() - }; - (slug, mode, source) - }) - .collect() -} - -fn module_source(slug: &str, functions: &[&CompiledFunction]) -> String { - let mut imports = Vec::new(); - for resolved in functions { - let function = &resolved.entry.spec.functions[resolved.function_index]; - imports.push(function.public_api.name.as_str()); - if let Some(result_class) = function.result_class() { - imports.push(result_class); - } - } - imports.sort_by_key(|name| natural_sort_key(name)); - imports.dedup(); - let imports = imports.join(", "); - let tests = functions - .iter() - .map(|resolved| { - let function = &resolved.entry.spec.functions[resolved.function_index]; - function_source(function) - }) - .collect::>() - .join("\n\n\n"); - - format!( - "{header}\nfrom __future__ import annotations\n\nimport pytest\n\nfrom _helpers import prepare_vector_case\nfrom ptfkit.{slug} import {imports}\n\n\n{tests}\n", - header = WRAPPER_HEADER.trim_end(), - ) -} - -fn function_source(function: &Function) -> String { - let cases_name = format!("CASES_{}", function.public_api.name.to_ascii_uppercase()); - let cases = function - .golden_tests - .iter() - .map(|case| { - format!( - " ({inputs}, {expected}, {rtol}, {atol}),", - inputs = dictionary(&case.inputs), - expected = dictionary(&case.expected), - rtol = float(case.rtol), - atol = float(case.atol), - ) - }) - .collect::>() - .join("\n"); - let assertion = expected_assertion(function, " ", ""); - let vector_tests = if !function.golden_tests.is_empty() { - vector_test_source(function, &cases_name) - } else { - Default::default() - }; - format!( - "{cases_name} = [\n{cases}\n]\n\n\n@pytest.mark.parametrize(('inputs', 'expected', 'rtol', 'atol'), {cases_name})\ndef test_{name}_golden(inputs: dict[str, float], expected: dict[str, float], rtol: float, atol: float):\n result = {name}(**inputs)\n\n{assertion}{vector_tests}", - cases_name = cases_name, - name = function.public_api.name, - assertion = assertion, - vector_tests = vector_tests, - ) -} - -fn vector_test_source(function: &Function, cases_name: &str) -> String { - let array_assertion = expected_assertion(function, " ", "[0]"); - let out_assertion = match &function.outputs { - Outputs::Scalar { .. } => " assert result is out", - Outputs::Record { .. } => { - " for actual, expected_out in zip(result, out, strict=True):\n assert actual is expected_out" - } - }; - let result_cls = function - .result_class() - .map(|result_class| format!(", {result_class}")) - .unwrap_or_default(); - format!( - r#" - - -def test_{name}_array(): - inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls}) - result = {name}(**inputs, out=None) -{array_assertion} - - -def test_{name}_out(): - inputs, expected, rtol, atol, out = prepare_vector_case({cases_name}{result_cls}) - result = {name}(**inputs, out=out) -{out_assertion} -{array_assertion}"#, - name = function.public_api.name, - cases_name = cases_name, - result_cls = result_cls, - array_assertion = array_assertion, - out_assertion = out_assertion, - ) -} - -fn expected_assertion(function: &Function, indent: &str, index: &str) -> String { - match &function.outputs { - Outputs::Scalar { field } => format!( - "{indent}assert result{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", - field.name - ), - Outputs::Record { fields, .. } => fields - .iter() - .map(|field| { - format!( - "{indent}assert result.{}{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", - field.name, field.name - ) - }) - .collect::>() - .join("\n"), - } -} - -fn dictionary(values: &BTreeMap) -> String { - let entries = values - .iter() - .map(|(name, value)| format!("'{name}': {}", float(*value))) - .collect::>() - .join(", "); - format!("{{{entries}}}") -} - -fn float(value: f64) -> String { - format!("{value:?}") -} diff --git a/codegen/src/targets/python/extension.rs b/codegen/src/targets/python/extension.rs new file mode 100644 index 0000000..e65e4fb --- /dev/null +++ b/codegen/src/targets/python/extension.rs @@ -0,0 +1,153 @@ +use anyhow::Result; + +use crate::{ + model::{CompiledFunction, Output}, + output::GeneratedFile, + render::{ + Writer, + c::{self, Dialect}, + }, + targets::group_by_source, +}; + +use super::C_HEADER; + +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { + let sources = group_by_source(functions); + let mut writes = Vec::new(); + for (slug, functions) in &sources { + let register = format!("ptfkit_register_{slug}"); + let mut source = Writer::new(); + source.write(C_HEADER); + source.write("#include \"ufunc.h\"\n\n"); + for function in functions { + source.write(ufunc(function)?); + } + source.line(format_args!("int {register}(PyObject *module) {{")); + source.indented(|writer| { + for function in functions { + writer.line(format_args!( + "if (ptfkit_add_ufunc(module, \"{name}\", {name}_functions, {name}_types, {nin}, {nout}) < 0) return -1;", + name = function.core.name, + nin = function.core.inputs.len(), + nout = output_count(&function.core.output), + )); + } + writer.line("return 0;"); + }); + source.line("}"); + writes.push(GeneratedFile::new( + format!("src/ptfkit/{slug}.c").into(), + source.into_string(), + )); + } + let mut entry = Writer::new(); + entry.write(C_HEADER); + entry.write( + r#"#define PY_SSIZE_T_CLEAN +#define PY_ARRAY_UNIQUE_SYMBOL PTFKIT_ARRAY_API +#include +#include +#include "#, + ); + entry.blank_line(); + for slug in sources.keys() { + entry.line(format_args!("#include \"{slug}.c\"")); + } + entry.write("\n\nstatic struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, \"_ptfkit\", NULL, -1, NULL };\n\nPyMODINIT_FUNC PyInit__ptfkit(void) {\n"); + entry.indented(|writer| { + writer.line("PyObject *module = PyModule_Create(&module_def);"); + writer.line("if (module == NULL) return NULL;"); + writer.line("import_array();"); + writer.line("import_ufunc();"); + for slug in sources.keys() { + writer.line(format_args!( + "if (ptfkit_register_{slug}(module) < 0) {{ Py_DECREF(module); return NULL; }}" + )); + } + writer.line("return module;"); + }); + entry.line("}"); + writes.push(GeneratedFile::new( + "src/ptfkit/ptfkit.c".into(), + entry.into_string(), + )); + Ok(writes) +} + +fn ufunc(function: &CompiledFunction) -> Result { + let name = &function.core.name; + let inputs = &function.core.inputs; + let values = match &function.core.output { + Output::Scalar => vec![ + function.entry.spec.functions[function.function_index] + .outputs + .fields()[0] + .name + .clone(), + ], + Output::Struct(fields) => fields.clone(), + }; + let types = std::iter::repeat_n("NPY_DOUBLE", inputs.len() + values.len()) + .collect::>() + .join(", "); + let mut writer = Writer::new(); + writer.line(format_args!("static void {name}_loop(char **args, const npy_intp *dimensions, const npy_intp *steps, void *data) {{")); + writer.indented(|writer| { + writer.line("npy_intp index;"); + writer.line("for (index = 0; index < dimensions[0]; index++) {"); + writer.indented(|writer| { + for (index, input) in inputs.iter().enumerate() { + writer.line(format_args!( + "const double {input} = *(const double *)args[{index}];" + )); + } + for variable in &function.ir.variables { + writer.write(format_args!("const double {} = ", variable.name)); + writer.write(c::expression( + &variable.expression, + inputs, + &function.ir.variables, + Dialect::C, + )); + writer.line(";"); + } + for (index, value) in values.iter().enumerate() { + writer.line(format_args!( + "*(double *)args[{}] = {value};", + inputs.len() + index + )); + } + writer.line(format_args!( + "for (int arg = 0; arg < {}; arg++) args[arg] += steps[arg];", + inputs.len() + values.len() + )); + }); + writer.line("}"); + }); + writer.line("}"); + writer.write(format_args!( + "static PyUFuncGenericFunction {name}_functions[] = {{ {name}_loop }};\nstatic char {name}_types[] = {{ {types} }};\n\n" + )); + Ok(writer.into_string()) +} + +fn output_count(output: &Output) -> usize { + match output { + Output::Scalar => 1, + Output::Struct(fields) => fields.len(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entry_source_initializes_the_private_module() { + let rendered = render(&[]).unwrap(); + let entry = &rendered.last().unwrap().contents; + assert!(entry.contains("PyInit__ptfkit")); + assert!(!entry.contains("#include \"ufunc.h\"")); + } +} diff --git a/codegen/src/targets/py/mod.rs b/codegen/src/targets/python/mod.rs similarity index 59% rename from codegen/src/targets/py/mod.rs rename to codegen/src/targets/python/mod.rs index fc6d99d..31637c0 100644 --- a/codegen/src/targets/py/mod.rs +++ b/codegen/src/targets/python/mod.rs @@ -1,30 +1,38 @@ -mod c; +mod extension; mod stub; +mod syntax; mod test; mod wrapper; -use crate::model::{CompiledFunction, PythonGeneration}; +use crate::model::CompiledFunction; + +use crate::output::GeneratedFile; pub(super) const C_HEADER: &str = "/* @generated by ptfkit-codegen; DO NOT EDIT. */\n"; pub(super) const WRAPPER_HEADER: &str = "# @generated by ptfkit-codegen; DO NOT EDIT.\n"; pub(super) struct Output { - pub(super) c_sources: Vec<(String, String)>, - pub(super) stub: String, - pub(super) tests: Vec<(String, PythonGeneration, String)>, - pub(super) wrappers: Vec<(String, PythonGeneration, String)>, + pub(super) extension: Vec, + pub(super) wrappers: Vec, + pub(super) tests: Vec, } pub(super) fn render(functions: &[CompiledFunction]) -> anyhow::Result { Ok(Output { - c_sources: c::render(functions)?, - stub: stub::render(functions), + extension: extension::render(functions)?, + wrappers: { + let mut wrappers = wrapper::render(functions)?; + wrappers.push(GeneratedFile::new( + "ptfkit/_ptfkit.pyi".into(), + stub::render(functions), + )); + wrappers + }, tests: test::render(functions), - wrappers: wrapper::render(functions)?, }) } -pub(super) fn natural_sort_key(value: &str) -> String { +pub(crate) fn natural_sort_key(value: &str) -> String { let mut key = String::new(); let mut digits = String::new(); for character in value.chars() { diff --git a/codegen/src/targets/py/stub.rs b/codegen/src/targets/python/stub.rs similarity index 58% rename from codegen/src/targets/py/stub.rs rename to codegen/src/targets/python/stub.rs index 36fcad1..ee37283 100644 --- a/codegen/src/targets/py/stub.rs +++ b/codegen/src/targets/python/stub.rs @@ -1,4 +1,6 @@ -use crate::{model::CompiledFunction, targets::py::WRAPPER_HEADER}; +use crate::model::CompiledFunction; + +use super::{WRAPPER_HEADER, syntax::Module}; pub(super) fn render(functions: &[CompiledFunction]) -> String { let mut names = functions @@ -7,14 +9,14 @@ pub(super) fn render(functions: &[CompiledFunction]) -> String { .collect::>(); names.sort_unstable(); - let mut lines = vec![ - WRAPPER_HEADER.trim_end().to_owned(), - String::new(), - "from numpy import ufunc".into(), - String::new(), - ]; - lines.extend(names.into_iter().map(|name| format!("{name}: ufunc"))); - format!("{}\n", lines.join("\n")) + let mut module = Module::new(WRAPPER_HEADER); + module.blank_line(); + module.import("numpy", "ufunc"); + module.blank_line(); + for name in names { + module.line(format_args!("{name}: ufunc")); + } + module.into_string() } #[cfg(test)] diff --git a/codegen/src/targets/python/syntax.rs b/codegen/src/targets/python/syntax.rs new file mode 100644 index 0000000..9a5b282 --- /dev/null +++ b/codegen/src/targets/python/syntax.rs @@ -0,0 +1,57 @@ +use std::fmt::Display; + +use crate::render::Writer; + +/// Small Python-specific rendering helpers for the declarations emitted by the +/// public wrapper and its generated tests. This deliberately models only the +/// constructs the generator owns, not Python's complete syntax. +pub(super) struct Module { + writer: Writer, +} + +impl Module { + pub(super) fn new(header: &str) -> Self { + let mut writer = Writer::new(); + writer.write(header.trim_end()); + Self { writer } + } + + pub(super) fn blank_line(&mut self) { + self.writer.blank_line(); + } + + pub(super) fn line(&mut self, value: impl Display) { + self.writer.line(value); + } + + pub(super) fn write(&mut self, value: impl Display) { + self.writer.write(value); + } + + pub(super) fn import(&mut self, module: &str, names: impl Display) { + self.line(format_args!("from {module} import {names}")); + } + + pub(super) fn assignment(&mut self, name: &str, value: impl Display) { + self.line(format_args!("{name} = {value}")); + } + + pub(super) fn block( + &mut self, + opening: impl Display, + render: impl FnOnce(&mut Writer), + closing: impl Display, + ) { + self.writer.line(opening); + self.writer.indented(render); + self.writer.line(closing); + } + + pub(super) fn indented(&mut self, render: impl FnOnce(&mut Writer)) { + self.writer.indented(render); + } + + pub(super) fn into_string(self) -> String { + self.writer.into_string() + } +} diff --git a/codegen/src/targets/python/test.rs b/codegen/src/targets/python/test.rs new file mode 100644 index 0000000..6523051 --- /dev/null +++ b/codegen/src/targets/python/test.rs @@ -0,0 +1,166 @@ +use std::collections::BTreeMap; + +use crate::{ + model::{CompiledFunction, Function, Outputs, PythonGeneration}, + output::GeneratedFile, +}; + +use super::{WRAPPER_HEADER, natural_sort_key, syntax::Module}; + +pub(super) fn render(functions: &[CompiledFunction]) -> Vec { + let mut modules: BTreeMap> = BTreeMap::new(); + for function in functions { + modules + .entry(function.entry.slug.clone()) + .or_default() + .push(function); + } + + modules + .into_iter() + .filter_map(|(slug, functions)| { + (functions[0].entry.spec.generation.public_python == PythonGeneration::Generated).then( + || { + GeneratedFile::new( + format!("tests/test_{slug}.py").into(), + module_source(&slug, &functions), + ) + }, + ) + }) + .collect() +} + +fn module_source(slug: &str, functions: &[&CompiledFunction]) -> String { + let mut imports = Vec::new(); + for resolved in functions { + let function = &resolved.entry.spec.functions[resolved.function_index]; + imports.push(function.public_api.name.as_str()); + if let Some(result_class) = function.result_class() { + imports.push(result_class); + } + } + imports.sort_by_key(|name| natural_sort_key(name)); + imports.dedup(); + let mut module = Module::new(WRAPPER_HEADER); + module.line("\nfrom __future__ import annotations"); + module.blank_line(); + module.line("import pytest"); + module.blank_line(); + module.import("_helpers", "prepare_vector_case"); + module.import(&format!("ptfkit.{slug}"), imports.join(", ")); + module.blank_line(); + module.blank_line(); + for (index, resolved) in functions.iter().enumerate() { + if index > 0 { + module.blank_line(); + module.blank_line(); + } + let function = &resolved.entry.spec.functions[resolved.function_index]; + function_source(&mut module, function); + } + module.into_string() +} + +fn function_source(module: &mut Module, function: &Function) { + let cases_name = format!("CASES_{}", function.public_api.name.to_ascii_uppercase()); + module.assignment(&cases_name, "["); + module.indented(|writer| { + for case in &function.golden_tests { + writer.line(format_args!( + "({}, {}, {}, {}),", + dictionary(&case.inputs), + dictionary(&case.expected), + float(case.rtol), + float(case.atol), + )); + } + }); + module.line("]"); + module.blank_line(); + module.blank_line(); + let name = &function.public_api.name; + module.line(format_args!( + "@pytest.mark.parametrize(('inputs', 'expected', 'rtol', 'atol'), {cases_name})" + )); + module.line(format_args!( + "def test_{name}_golden(inputs: dict[str, float], expected: dict[str, float], rtol: float, atol: float):" + )); + module.indented(|writer| { + writer.line(format_args!("result = {name}(**inputs)")); + writer.blank_line(); + render_expected_assertion(writer, function, ""); + }); + if !function.golden_tests.is_empty() { + vector_test_source(module, function, &cases_name); + } +} + +fn vector_test_source(module: &mut Module, function: &Function, cases_name: &str) { + let result_cls = function + .result_class() + .map(|result_class| format!(", {result_class}")) + .unwrap_or_default(); + let name = &function.public_api.name; + module.blank_line(); + module.blank_line(); + module.line(format_args!("def test_{name}_array():")); + module.indented(|writer| { + writer.line(format_args!( + "inputs, expected, rtol, atol, _out = prepare_vector_case({cases_name}{result_cls})" + )); + writer.line(format_args!("result = {name}(**inputs, out=None)")); + render_expected_assertion(writer, function, "[0]"); + }); + module.blank_line(); + module.blank_line(); + module.line(format_args!("def test_{name}_out():")); + module.indented(|writer| { + writer.line(format_args!( + "inputs, expected, rtol, atol, out = prepare_vector_case({cases_name}{result_cls})" + )); + writer.line(format_args!("result = {name}(**inputs, out=out)")); + render_out_assertion(writer, function); + render_expected_assertion(writer, function, "[0]"); + }); +} + +fn render_expected_assertion(writer: &mut crate::render::Writer, function: &Function, index: &str) { + match &function.outputs { + Outputs::Scalar { field } => writer.line(format_args!( + "assert result{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", + field.name + )), + Outputs::Record { fields, .. } => { + for field in fields { + writer.line(format_args!( + "assert result.{}{index} == pytest.approx(expected['{}'], rel=rtol, abs=atol)", + field.name, field.name + )); + } + } + } +} + +fn render_out_assertion(writer: &mut crate::render::Writer, function: &Function) { + match &function.outputs { + Outputs::Scalar { .. } => writer.line("assert result is out"), + Outputs::Record { .. } => { + writer.line("for actual, expected_out in zip(result, out, strict=True):"); + writer.indented(|writer| writer.line("assert actual is expected_out")); + } + } +} + +fn dictionary(values: &BTreeMap) -> String { + let entries = values + .iter() + .map(|(name, value)| format!("'{name}': {}", float(*value))) + .collect::>() + .join(", "); + format!("{{{entries}}}") +} + +fn float(value: f64) -> String { + format!("{value:?}") +} diff --git a/codegen/src/targets/py/wrapper.rs b/codegen/src/targets/python/wrapper.rs similarity index 51% rename from codegen/src/targets/py/wrapper.rs rename to codegen/src/targets/python/wrapper.rs index 3a326cb..3a08d34 100644 --- a/codegen/src/targets/py/wrapper.rs +++ b/codegen/src/targets/python/wrapper.rs @@ -1,10 +1,17 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, path::PathBuf}; use anyhow::{Result, bail}; -use crate::model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}; +use crate::{ + documentation::{self as docs, FunctionDocument}, + model::{CompiledFunction, Function, Parameter, PythonGeneration, Scope, Source}, + output::GeneratedFile, + render::Writer, +}; -use super::{super::documentation, WRAPPER_HEADER, natural_sort_key}; +use super::{WRAPPER_HEADER, natural_sort_key, syntax::Module}; + +const LINE_WIDTH: usize = 100; struct PythonFunction<'a> { name: &'a str, @@ -14,18 +21,22 @@ struct PythonFunction<'a> { array_inputs: Vec, keyword_inputs: Vec, parameters: String, - docstring: String, + docstring: PythonDocstring, +} + +#[derive(PartialEq, Eq)] +struct PythonDocstring { + summary: String, + sections: Vec<(&'static str, Vec)>, } struct PythonResultClass { name: String, field_definitions: String, - docstring: String, + docstring: PythonDocstring, } -pub(crate) fn render( - functions: &[CompiledFunction], -) -> Result> { +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { let mut modules: BTreeMap> = BTreeMap::new(); for function in functions { modules @@ -38,7 +49,6 @@ pub(crate) fn render( for (module, functions) in modules { let mode = functions[0].entry.spec.generation.public_python; if mode == PythonGeneration::Manual { - generated.push((module, mode, String::new())); continue; } let source = &functions[0].entry.spec.source; @@ -53,7 +63,7 @@ pub(crate) fn render( .outputs .fields() .iter() - .map(|output| format!(" {}: T", output.name)) + .map(|output| format!("{}: T", output.name)) .collect::>() .join("\n"), docstring: result_class_docstring(function), @@ -91,7 +101,10 @@ pub(crate) fn render( &functions, &exports, ); - generated.push((module, mode, text)); + generated.push(GeneratedFile::new( + PathBuf::from(module.replace('.', "/")).with_extension("py"), + text, + )); } Ok(generated) } @@ -104,7 +117,7 @@ fn module_source( functions: &[PythonFunction<'_>], exports: &[String], ) -> String { - let mut sections = vec![WRAPPER_HEADER.trim_end().into()]; + let mut module = Module::new(WRAPPER_HEADER); if functions.len() > 1 { let has_long_import = functions.iter().any(|function| { format!( @@ -113,53 +126,76 @@ fn module_source( ) .chars() .count() - > 100 + > LINE_WIDTH }); - sections.push(if has_long_import { - "# ruff: noqa: E501, I001".into() + module.blank_line(); + module.line(if has_long_import { + "# ruff: noqa: E501, I001" } else { - "# ruff: noqa: I001".into() + "# ruff: noqa: I001" }); } - sections.extend([ - module_docstring(source, scope), - "from __future__ import annotations".into(), - format!("from typing import {typing_imports}"), - "from ptfkit._ptfkit import (".into(), - ]); - sections.extend(functions.iter().map(|function| { - format!( - " {rust_name} as _{rust_name},", - rust_name = function.rust_name - ) - })); - sections.extend([ - ")\n".into(), - "if TYPE_CHECKING:\n from numpy import floating\n from numpy.typing import ArrayLike, NDArray" - .into(), - ]); + module.blank_line(); + render_module_docstring(&mut module, source, scope); + module.write(format_args!( + "from __future__ import annotations\n\nfrom typing import {typing_imports}\n\n" + )); + module.block( + "from ptfkit._ptfkit import (", + |writer| { + for function in functions { + writer.line(format_args!( + "{rust_name} as _{rust_name},", + rust_name = function.rust_name + )); + } + }, + ")", + ); + module.line("\n\nif TYPE_CHECKING:"); + module.indented(|writer| { + writer.line("from numpy import floating"); + writer.line("from numpy.typing import ArrayLike, NDArray"); + }); if !classes.is_empty() { - sections.push("T = TypeVar('T')".into()); - sections.extend(classes.iter().map(|class| { - format!( - "class {}(NamedTuple, Generic[T]):\n{}\n{}", - class.name, class.docstring, class.field_definitions - ) - })); + module.blank_line(); + module.assignment("T", "TypeVar('T')"); + for class in classes { + module.blank_line(); + module.line(format_args!( + "class {}(NamedTuple, Generic[T]):", + class.name + )); + module.indented(|writer| { + render_docstring(writer, &class.docstring, 4); + for line in class.field_definitions.lines() { + writer.line(line); + } + }); + } } - sections.push(format!( - "__all__ = [{}]", - exports - .iter() - .map(|export| format!("'{export}'")) - .collect::>() - .join(", ") - )); - sections.extend(functions.iter().map(function_source)); - format!("{}\n", sections.join("\n\n")) + module.blank_line(); + module.assignment( + "__all__", + format_args!( + "[{}]", + exports + .iter() + .map(|export| format!("'{export}'")) + .collect::>() + .join(", ") + ), + ); + for function in functions { + module.blank_line(); + module.blank_line(); + render_function(&mut module, function); + } + module.line(""); + module.into_string() } -fn function_source(function: &PythonFunction<'_>) -> String { +fn render_function(module: &mut Module, function: &PythonFunction<'_>) { let scalar_result = function .result_class .map(|class| format!("{class}[floating]")) @@ -178,20 +214,51 @@ fn function_source(function: &PythonFunction<'_>) -> String { } else { "return values".into() }; - let calculation = format!( - " if out is None:\n values = _{}({})\n else:\n values = _{}({}, out={out})", - function.rust_name, function.parameters, function.rust_name, function.parameters - ); - format!( - "@overload\ndef {}(*, {}) -> {scalar_result}: ...\n\n@overload\ndef {}(*, {},\n out: {array_result} | None = None,\n) -> {array_result}: ...\n\ndef {}(*,\n{}\n out: {array_result} | None = None,\n) -> {scalar_result} | {array_result}:\n{}\n{calculation}\n\n {result}", + module.line("@overload"); + module.line(format_args!( + "def {}(*, {}) -> {scalar_result}: ...", function.name, function.scalar_inputs.join(", "), + )); + module.blank_line(); + module.line("@overload"); + module.line(format_args!( + "def {}(*, {},", function.name, function.array_inputs.join(", "), - function.name, - function.keyword_inputs.join("\n"), - function.docstring, - ) + )); + module.indented(|writer| { + writer.line(format_args!("out: {array_result} | None = None,")); + }); + module.line(format_args!(") -> {array_result}: ...")); + module.blank_line(); + module.line(format_args!("def {}(*,", function.name)); + module.indented(|writer| { + for input in &function.keyword_inputs { + writer.line(input); + } + writer.line(format_args!("out: {array_result} | None = None,")); + }); + module.line(format_args!(") -> {scalar_result} | {array_result}:")); + module.indented(|writer| { + render_docstring(writer, &function.docstring, 4); + writer.line("if out is None:"); + writer.indented(|writer| { + writer.line(format_args!( + "values = _{}({})", + function.rust_name, function.parameters + )); + }); + writer.line("else:"); + writer.indented(|writer| { + writer.line(format_args!( + "values = _{}({}, out={out})", + function.rust_name, function.parameters + )); + }); + writer.blank_line(); + writer.line(result); + }); } fn view(resolved: &CompiledFunction) -> PythonFunction<'_> { @@ -213,73 +280,73 @@ fn view(resolved: &CompiledFunction) -> PythonFunction<'_> { keyword_inputs: function .inputs .iter() - .map(|input| format!(" {}: float | ArrayLike,", input.name)) + .map(|input| format!("{}: float | ArrayLike,", input.name)) .collect(), parameters: names.join(", "), docstring: function_docstring(function), } } -fn module_docstring(source: &Source, scope: &Scope) -> String { - let mut lines = vec![ - format!("r\"\"\"{}", source.summary), - String::new(), - "Reference:".into(), - ]; - lines.extend(wrap_markdown_block(&source.citation_apa, " ")); - if let Some(doi) = &source.doi { - lines.extend(wrap_markdown_block( +fn render_module_docstring(module: &mut Module, source: &Source, scope: &Scope) { + let document = docs::for_source(source, scope); + module.line(format_args!("r\"\"\"{}", document.summary)); + module.blank_line(); + module.line("Reference:"); + render_markdown_block(module, document.reference.citation, " "); + if let Some(doi) = document.reference.doi { + render_markdown_block( + module, &format!("[DOI: {}]({})", doi.identifier, doi.url), " ", - )); + ); } - if let Some(territory) = &scope.territory { - definition_list_block(&mut lines, "Territory", territory); + if let Some(territory) = document.territory { + render_definition_list_block(module, "Territory", territory); } - if let Some(dataset) = &scope.dataset { - definition_list_block(&mut lines, "Dataset", dataset); + if let Some(dataset) = document.dataset { + render_definition_list_block(module, "Dataset", dataset); } - lines.push(String::new()); - lines.push("\"\"\"".into()); - lines.join("\n") + module.blank_line(); + module.line("\"\"\""); } -fn function_docstring(function: &Function) -> String { - let mut arguments = function - .inputs +fn function_docstring(function: &Function) -> PythonDocstring { + let document = docs::for_function(function); + function_docstring_from_document(document, function.result_class()) +} + +fn function_docstring_from_document( + document: FunctionDocument<'_>, + result_class: Option<&str>, +) -> PythonDocstring { + let mut arguments = document + .parameters .iter() .map(parameter_documentation) .collect::>(); arguments.push("out: Optional output arrays for in-place calculation.".into()); - let returns = if let Some(result_class) = function.result_class() { + let returns = if let Some(result_class) = result_class { vec![format!( "{result_class}: Results grouped by result attributes." )] } else { - function - .outputs - .fields() - .iter() - .map(parameter_documentation) - .collect() + match document.returns { + docs::Returns::Scalar(field) => vec![parameter_documentation(field)], + docs::Returns::Record { fields, .. } => { + fields.iter().map(parameter_documentation).collect() + } + } }; let mut sections = vec![("Arguments", arguments), ("Returns", returns)]; - if let Some(territory) = &function.scope.territory { - sections.push(("Territory", vec![territory.clone()])); + if let Some(territory) = document.territory { + sections.push(("Territory", vec![territory.into()])); } let models = [ - function - .scope + document .models .h_theta - .as_ref() .map(|model| format!("$h(\\theta)$: {model}")), - function - .scope - .models - .k_h - .as_ref() - .map(|model| format!("$k(h)$: {model}")), + document.models.k_h.map(|model| format!("$k(h)$: {model}")), ] .into_iter() .flatten() @@ -291,22 +358,24 @@ fn function_docstring(function: &Function) -> String { "Notes", std::iter::once(format!( "Prediction target: {}", - function.scope.prediction_target + document.remarks.prediction_target )) - .chain(function.documentation.notes.iter().cloned()) + .chain(document.notes.iter().cloned()) .collect(), )); - if !function.documentation.warnings.is_empty() { - sections.push(("Warning", function.documentation.warnings.clone())); + if !document.warnings.is_empty() { + sections.push(("Warning", document.warnings.to_vec())); + } + PythonDocstring { + summary: document.summary.to_owned(), + sections, } - render_docstring(" ", &function.public_api.summary, §ions, 4) } -fn result_class_docstring(function: &Function) -> String { - render_docstring( - " ", - "Results returned by the matching PTF.", - &[( +fn result_class_docstring(function: &Function) -> PythonDocstring { + PythonDocstring { + summary: "Results returned by the matching PTF.".into(), + sections: vec![( "Attributes", function .outputs @@ -315,28 +384,27 @@ fn result_class_docstring(function: &Function) -> String { .map(parameter_documentation) .collect(), )], - 4, - ) + } } fn parameter_documentation(parameter: &Parameter) -> String { - documentation::parameter_documentation(parameter) + docs::parameter_documentation(parameter) } -fn definition_list_block(lines: &mut Vec, name: &str, text: &str) { - lines.push(String::new()); - lines.push(name.into()); - lines.push(String::new()); - let wrapped = wrap_markdown_block(text, ": "); - lines.extend(wrapped); +fn render_definition_list_block(module: &mut Module, name: &str, text: &str) { + module.blank_line(); + module.line(name); + module.blank_line(); + render_markdown_block(module, text, ": "); } -fn wrap_markdown_block(text: &str, first_prefix: &str) -> Vec { - let first_width = 100 - first_prefix.len(); +fn render_markdown_block(module: &mut Module, text: &str, first_prefix: &str) { + let first_width = LINE_WIDTH - first_prefix.len(); let wrapped = wrap_doc_line(text, first_width, 96); - let mut lines = vec![format!("{first_prefix}{}", wrapped[0])]; - lines.extend(wrapped.iter().skip(1).map(|line| format!(" {line}"))); - lines + module.line(format_args!("{first_prefix}{}", wrapped[0])); + for line in wrapped.iter().skip(1) { + module.line(format_args!(" {line}")); + } } fn with_terminal_punctuation(text: &str) -> String { @@ -347,64 +415,52 @@ fn with_terminal_punctuation(text: &str) -> String { } } -fn render_docstring( - indent: &str, - summary: &str, - sections: &[(&str, Vec)], - indent_width: usize, -) -> String { - let raw = summary.contains('\\') - || sections +fn render_docstring(writer: &mut Writer, docstring: &PythonDocstring, indent_width: usize) { + let left_width = LINE_WIDTH - indent_width; + let raw = docstring.summary.contains('\\') + || docstring + .sections .iter() .flat_map(|(title, entries)| { std::iter::once(*title).chain(entries.iter().map(String::as_str)) }) .any(|text| text.contains('\\')); let opening = if raw { "r\"\"\"" } else { "\"\"\"" }; - let summary = with_terminal_punctuation(summary); - let first_line_width = 100 - indent_width - 3; + let summary = with_terminal_punctuation(&docstring.summary); + let first_line_width = left_width - 3; let (summary, description) = if summary.len() <= first_line_width { (summary, None) } else { ( "Calculate the pedotransfer function.".into(), - Some(wrap_doc_line( - &summary, - 100 - indent_width, - 100 - indent_width, - )), + Some(wrap_doc_line(&summary, left_width, left_width)), ) }; - let mut lines = vec![format!("{indent}{opening}{summary}"), String::new()]; + writer.line(format_args!("{opening}{summary}")); + writer.blank_line(); if let Some(description) = description { - lines.extend( - description - .into_iter() - .map(|line| format!("{indent}{line}")), - ); - lines.push(String::new()); + for line in description { + writer.line(line); + } + writer.blank_line(); } - for (section_index, (title, entries)) in sections.iter().enumerate() { + for (section_index, (title, entries)) in docstring.sections.iter().enumerate() { if !title.is_empty() { - lines.push(format!("{indent}{title}:")); + writer.line(format_args!("{title}:")); } for entry in entries { - let wrapped = wrap_doc_line(entry, 100 - indent_width - 4, 100 - indent_width - 8); - lines.push(format!("{indent} {}", wrapped[0])); - lines.extend( - wrapped - .iter() - .skip(1) - .map(|line| format!("{indent} {line}")), - ); + let wrapped = wrap_doc_line(entry, left_width - 4, left_width - 8); + writer.line(format_args!(" {}", wrapped[0])); + for line in wrapped.iter().skip(1) { + writer.line(format_args!(" {line}")); + } } - if section_index + 1 != sections.len() { - lines.push(String::new()); + if section_index + 1 != docstring.sections.len() { + writer.blank_line(); } } - lines.push(String::new()); - lines.push(format!("{indent}\"\"\"")); - lines.join("\n") + writer.blank_line(); + writer.line("\"\"\""); } fn wrap_doc_line(text: &str, first_width: usize, continuation_width: usize) -> Vec { @@ -430,8 +486,11 @@ fn wrap_doc_line(text: &str, first_width: usize, continuation_width: usize) -> V #[cfg(test)] mod tests { - use super::{function_docstring, module_docstring}; - use crate::model::{Documentation, Function, FunctionScope, Models, PublicApi, Scope, Source}; + use super::{function_docstring, render_module_docstring}; + use crate::{ + model::{Documentation, Function, FunctionScope, Models, PublicApi, Scope, Source}, + targets::python::syntax::Module, + }; fn function(territory: Option<&str>) -> Function { Function { @@ -460,11 +519,22 @@ mod tests { #[test] fn function_territory_is_rendered_only_when_declared() { + let with_territory = function_docstring(&function(Some("Narrow test region."))); + assert_eq!( + with_territory + .sections + .iter() + .find(|(title, _)| *title == "Territory") + .map(|(_, entries)| entries), + Some(&vec!["Narrow test region.".into()]) + ); + let without_territory = function_docstring(&function(None)); assert!( - function_docstring(&function(Some("Narrow test region."))) - .contains("Territory:\n Narrow test region.") + without_territory + .sections + .iter() + .all(|(title, _)| *title != "Territory") ); - assert!(!function_docstring(&function(None)).contains("Territory:")); } #[test] @@ -482,7 +552,9 @@ mod tests { dataset: None, }; - let docstring = module_docstring(&source, &scope); + let mut module = Module::new(""); + render_module_docstring(&mut module, &source, &scope); + let docstring = module.into_string(); assert_eq!( docstring.lines().next(), Some("r\"\"\"Test et al. (2026), short territory.") diff --git a/codegen/src/targets/reference/c.rs b/codegen/src/targets/reference/c.rs new file mode 100644 index 0000000..2f7f24e --- /dev/null +++ b/codegen/src/targets/reference/c.rs @@ -0,0 +1,396 @@ +use std::{collections::BTreeSet, path::PathBuf}; + +use anyhow::Result; + +use crate::render::markdown::HEADER; +use crate::{ + documentation::{self as docs}, + model::{CompiledFunction, Output, Parameter}, + output::GeneratedFile, + render::{Writer, markdown}, + targets::{group_by_source, native::c_result_name}, +}; + +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { + let sources = group_by_source(functions); + let mut files = vec![markdown::markdown_file("index.md", |writer| { + render_index(writer, &sources); + })]; + files.push(header_file("headers/ptfkit.md", "ptfkit", |writer| { + render_umbrella(writer, &sources); + Ok(()) + })?); + + let mut index_entries = Vec::new(); + for (slug, functions) in sources { + files.push(header_file(format!("headers/{slug}.md"), slug, |writer| { + render_header(writer, slug, &functions) + })?); + for function in functions { + index_entries.push((slug, function)); + } + } + index_entries.sort_by_key(|(_, function)| natural_sort_key(&function.core.name)); + files.push(markdown::markdown_file("functions.md", |writer| { + render_functions_index(writer, &index_entries); + })); + Ok(files) +} + +fn header_file( + path: impl Into, + slug: &str, + render: impl FnOnce(&mut Writer) -> Result<()>, +) -> Result { + let mut writer = Writer::new(); + markdown::frontmatter(&mut writer, |writer| { + writer.line(format_args!("title: \"{slug}.h\"")); + }); + render(&mut writer)?; + Ok(GeneratedFile::new( + path.into(), + markdown::markdown_contents(writer), + )) +} + +fn render_index( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C API reference"); + }); + writer.write( + "# C API reference\n\nptfkit's C API is organized around installed headers.\n\n## Headers\n\n- [``](headers/ptfkit.md) — Aggregates every ptfkit source header.\n", + ); + for (slug, functions) in sources { + let summary = docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope, + ) + .summary; + writer.line(format_args!( + "- [``](headers/{slug}.md) — {}", + escape_text(summary) + )); + } + writer.blank_line(); + writer.line("See the [function index](functions.md) for all public C functions."); +} + +fn render_umbrella( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + writer.write(HEADER); + writer.write("# ``\n\n"); + markdown::code_block(writer, "c", |writer| { + writer.line("#include "); + }); + writer.write( + "This umbrella header aggregates every public ptfkit source header. Include an individual header when only one source is needed.\n\n## Included headers\n\n", + ); + for (slug, functions) in sources { + writer.line(format_args!( + "- [``]({slug}.md) — {}", + escape_text( + docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope + ) + .summary, + ) + )); + } +} + +fn render_header(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction]) -> Result<()> { + let first = functions + .first() + .expect("compiled source contains at least one function"); + let source = docs::for_source(&first.entry.spec.source, &first.entry.spec.scope); + writer.write(HEADER); + writer.write(format_args!("# ``\n\n")); + markdown::code_block(writer, "c", |writer| { + writer.line(format_args!("#include ")); + }); + writer.write(format_args!( + "{}\n\n## Source\n\n{}\n\n", + escape_text(source.summary), + escape_text(source.reference.citation), + )); + if let Some(doi) = source.reference.doi { + writer.write(format_args!( + "[DOI: {}]({})\n\n", + escape_text(doi.identifier), + doi.url + )); + } + if source.territory.is_some() || source.dataset.is_some() { + writer.write("## Scope\n\n"); + if let Some(territory) = source.territory { + writer.write(format_args!( + "**Territory:** {}\n\n", + escape_text(territory) + )); + } + if let Some(dataset) = source.dataset { + writer.write(format_args!("**Dataset:** {}\n\n", escape_text(dataset))); + } + } + writer.write(format_args!( + "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" + )); + + let mut structures = BTreeSet::new(); + for function in functions { + if let Output::Struct(_) = &function.core.output { + let spec = spec(function); + let name = spec + .result_class() + .expect("record output has a result class"); + let c_name = c_result_name(name); + if structures.insert(c_name.clone()) { + render_structure(writer, &c_name, spec.outputs.fields()); + } + } + } + writer.write("## Functions\n\n"); + for function in functions { + render_function_documentation(writer, function)?; + } + Ok(()) +} + +fn render_structure(writer: &mut Writer, name: &str, fields: &[Parameter]) { + writer.write(format_args!("## `{name}`\n\n")); + markdown::code_block(writer, "c", |writer| { + writer.line("typedef struct {"); + writer.indented(|writer| { + for field in fields { + writer.line(format_args!("double {};", field.name)); + } + }); + writer.line(format_args!(" }} {name};")); + }); + writer.line("| Field | Description |"); + writer.line("| --- | --- |"); + for field in fields { + writer.line(format_args!( + "| `{}` | {} |", + field.name, + parameter_details(field) + )); + } + writer.blank_line(); +} + +fn render_function_documentation(writer: &mut Writer, function: &CompiledFunction) -> Result<()> { + let spec = spec(function); + let document = docs::for_function(spec); + let anchor = function_anchor(&function.core.name); + writer.write(format_args!( + "### `{}` {{#{anchor}}}\n\n{}\n\n", + function.core.name, + escape_text(document.summary), + )); + let signature = signature(function)?; + markdown::code_block(writer, "c", |writer| { + writer.line(format_args!("{signature};")); + }); + writer.write("#### Parameters\n\n| Name | Direction | Description |\n| --- | --- | --- |\n"); + for parameter in document.parameters { + writer.line(format_args!( + "| `{}` | in | {} |", + parameter.name, + parameter_details(parameter) + )); + } + writer.blank_line(); + writer.write("#### Returns\n\n"); + match document.returns { + docs::Returns::Scalar(field) => { + writer.write(format_args!("{}\n\n", parameter_details(field))); + } + docs::Returns::Record { .. } => { + let name = spec + .result_class() + .expect("record output has a result class"); + writer.write(format_args!("A `{}` value.\n\n", c_result_name(name))); + } + } + for note in document.notes { + render_admonition(writer, "note", note); + } + for warning in document.warnings { + render_admonition(writer, "warning", warning); + } + Ok(()) +} + +fn render_functions_index(writer: &mut Writer, functions: &[(&str, &CompiledFunction)]) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C function index"); + }); + writer.write("# C function index\n\n| Function | Summary | Header |\n| --- | --- | --- |\n"); + for (slug, function) in functions { + writer.line(format_args!( + "| [`{0}`](headers/{1}.md#{2}) | {3} | [``](headers/{1}.md) |", + function.core.name, + slug, + function_anchor(&function.core.name), + escape_table(docs::for_function(spec(function)).summary), + )); + } +} + +fn signature(function: &CompiledFunction) -> Result { + let spec = spec(function); + let result = match &function.core.output { + Output::Scalar => "double".to_owned(), + Output::Struct(_) => c_result_name( + spec.result_class() + .expect("record output has a result class"), + ), + }; + Ok(format!( + "static inline {result} {}({})", + function.core.name, + function + .core + .inputs + .iter() + .map(|input| format!("double {input}")) + .collect::>() + .join(", ") + )) +} + +fn spec(function: &CompiledFunction) -> &crate::model::Function { + &function.entry.spec.functions[function.function_index] +} + +fn function_anchor(name: &str) -> String { + format!("function-{name}") +} + +fn parameter_details(parameter: &Parameter) -> String { + format!( + "{} ({})", + escape_table(¶meter.description), + escape_table(¶meter.unit) + ) +} + +fn render_admonition(writer: &mut Writer, kind: &str, body: &str) { + markdown::admonition(writer, kind, body, escape_text); +} + +fn escape_text(value: &str) -> String { + value.replace('\\', "\\\\").replace('`', "\\`") +} + +fn escape_table(value: &str) -> String { + escape_text(value).replace('\n', " ").replace('|', "\\|") +} + +fn natural_sort_key(value: &str) -> String { + crate::targets::python::natural_sort_key(value) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + + fn rendered_files() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("codegen directory has a repository parent"); + let entries = crate::specs::load(root).expect("repository specifications load"); + let compiled = + crate::compile::functions(entries).expect("repository specifications compile"); + render(&compiled).expect("C documentation renders") + } + + fn contents<'a>(files: &'a [GeneratedFile], path: &str) -> &'a str { + &files + .iter() + .find(|file| file.path == Path::new(path)) + .unwrap_or_else(|| panic!("missing generated file {path}")) + .contents + } + + #[test] + fn documents_headers_functions_and_record_fields_from_compiled_sources() { + let files = rendered_files(); + let index = contents(&files, "index.md"); + let rawls = contents(&files, "headers/rawls1982.md"); + let function_index = contents(&files, "functions.md"); + + assert!(index.starts_with( + "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: C API reference\n---\n" + )); + assert!(rawls.starts_with("---\ntitle: \"rawls1982.h\"\n---\n")); + assert!(index.contains("[``](headers/ptfkit.md)")); + assert!(rawls.contains("## `rawls1982_ptf_result`")); + assert!(rawls.contains("| `theta_4` | Volumetric water content at -4 kPa. (cm^3/cm^3) |")); + assert!(rawls.contains( + "static inline double calc_ptf_rawls1982_theta_1500(double clay, double organic_matter)" + )); + assert!(rawls.contains("static inline rawls1982_ptf_result calc_ptf_rawls1982_full_wrc")); + assert!(rawls.contains("{#function-calc_ptf_rawls1982_theta_1500}")); + assert!( + function_index.contains("headers/rawls1982.md#function-calc_ptf_rawls1982_theta_1500") + ); + assert!(rawls.contains("[PTF catalog page](../../../ptf-catalog/sources/rawls1982.md)")); + } + + #[test] + fn umbrella_lists_each_source_header_once_in_natural_order() { + let files = rendered_files(); + let umbrella = contents(&files, "headers/ptfkit.md"); + let index = contents(&files, "index.md"); + let source_headers = files + .iter() + .filter(|file| { + file.path.starts_with("headers") && file.path != Path::new("headers/ptfkit.md") + }) + .count(); + + assert_eq!(umbrella.matches("- [` Result> { + let sources = group_by_source(functions); + let mut files = vec![markdown::markdown_file("index.md", |writer| { + render_index(writer, &sources); + })]; + files.push(markdown::markdown_file("modules/ptfkit.md", |writer| { + render_umbrella(writer, &sources); + })); + + let mut index_entries = Vec::new(); + for (slug, functions) in sources { + files.push(module_file(slug, &functions)?); + for function in functions { + index_entries.push((slug, function)); + } + } + index_entries.sort_by_key(|(_, function)| natural_sort_key(&function.core.name)); + files.push(markdown::markdown_file("functions.md", |writer| { + render_functions_index(writer, &index_entries); + })); + Ok(files) +} + +fn module_file(slug: &str, functions: &[&CompiledFunction]) -> Result { + let mut writer = Writer::new(); + render_module(&mut writer, slug, functions)?; + Ok(GeneratedFile::new( + PathBuf::from(format!("modules/{slug}.md")), + markdown::markdown_contents(writer), + )) +} + +fn render_index( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C++ API reference"); + }); + writer.write( + "# C++ API reference\n\nptfkit's C++ API is organized around C++20 modules.\n\n## Modules\n\n- [`ptfkit`](modules/ptfkit.md) — Re-exports every ptfkit source module.\n", + ); + for (slug, functions) in sources { + writer.line(format_args!( + "- [`ptfkit.{slug}`](modules/{slug}.md) — {}", + escape_text( + docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope + ) + .summary, + ) + )); + } + writer.blank_line(); + writer.line("See the [function index](functions.md) for all public C++ functions."); +} + +fn render_umbrella( + writer: &mut Writer, + sources: &std::collections::BTreeMap<&str, Vec<&CompiledFunction>>, +) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C++ module ptfkit"); + writer.line("nav-title: ptfkit"); + }); + writer.write("# `ptfkit`\n\n"); + markdown::code_block(writer, "cpp", |writer| { + writer.line("import ptfkit;"); + }); + writer.write( + "This umbrella module re-exports every public ptfkit source module. Import an individual module when only one source is needed.\n\n## Re-exported modules\n\n", + ); + for (slug, functions) in sources { + writer.line(format_args!( + "- [`ptfkit.{slug}`]({slug}.md) — {}", + escape_text( + docs::for_source( + &functions[0].entry.spec.source, + &functions[0].entry.spec.scope + ) + .summary, + ) + )); + } +} + +fn render_module(writer: &mut Writer, slug: &str, functions: &[&CompiledFunction]) -> Result<()> { + let first = functions + .first() + .expect("compiled source contains at least one function"); + let source = docs::for_source(&first.entry.spec.source, &first.entry.spec.scope); + markdown::generated_frontmatter(writer, |writer| { + writer.line(format_args!("title: C++ module ptfkit.{slug}")); + writer.line(format_args!("nav-title: ptfkit.{slug}")); + }); + writer.write(format_args!("# `ptfkit.{slug}`\n\n")); + markdown::code_block(writer, "cpp", |writer| { + writer.line(format_args!("import ptfkit.{slug};")); + }); + writer.write(format_args!( + "**Exported namespace:** `ptfkit::{slug}`\n\n{}\n\n## Source\n\n{}\n\n", + escape_text(source.summary), + escape_text(source.reference.citation), + )); + if let Some(doi) = source.reference.doi { + writer.write(format_args!( + "[DOI: {}]({})\n\n", + escape_text(doi.identifier), + doi.url + )); + } + if source.territory.is_some() || source.dataset.is_some() { + writer.write("## Scope\n\n"); + if let Some(territory) = source.territory { + writer.write(format_args!( + "**Territory:** {}\n\n", + escape_text(territory) + )); + } + if let Some(dataset) = source.dataset { + writer.write(format_args!("**Dataset:** {}\n\n", escape_text(dataset))); + } + } + writer.write(format_args!( + "[PTF catalog page](../../../ptf-catalog/sources/{slug}.md)\n\n" + )); + + let mut structures = BTreeSet::new(); + for function in functions { + if let Output::Struct(_) = &function.core.output { + let result = result_class(function)?; + if structures.insert(result) { + render_structure(writer, result, spec(function).outputs.fields()); + } + } + } + writer.write("## Functions\n\n"); + for function in functions { + render_function_documentation(writer, function)?; + } + Ok(()) +} + +fn render_structure(writer: &mut Writer, name: &str, fields: &[Parameter]) { + writer.write(format_args!("## `{name}`\n\n")); + markdown::code_block(writer, "cpp", |writer| { + writer.line(format_args!("struct {name} {{")); + writer.indented(|writer| { + for field in fields { + writer.line(format_args!("double {};", field.name)); + } + }); + writer.line("};"); + }); + writer.line("| Field | Description |"); + writer.line("| --- | --- |"); + for field in fields { + writer.line(format_args!( + "| `{}` | {} |", + field.name, + parameter_details(field) + )); + } + writer.blank_line(); +} + +fn render_function_documentation(writer: &mut Writer, function: &CompiledFunction) -> Result<()> { + let spec = spec(function); + let document = docs::for_function(spec); + let anchor = function_anchor(&function.core.name); + writer.write(format_args!( + "### `{}` {{#{anchor}}}\n\n{}\n\n", + function.core.name, + escape_text(document.summary), + )); + let signature = signature(function)?; + markdown::code_block(writer, "cpp", |writer| { + writer.line(signature); + }); + writer.write("#### Parameters\n\n| Name | Description |\n| --- | --- |\n"); + for parameter in document.parameters { + writer.line(format_args!( + "| `{}` | {} |", + parameter.name, + parameter_details(parameter) + )); + } + writer.blank_line(); + writer.write("#### Returns\n\n"); + match document.returns { + docs::Returns::Scalar(field) => { + writer.write(format_args!("{}\n\n", parameter_details(field))); + } + docs::Returns::Record { .. } => { + writer.write(format_args!("A `{}` value.\n\n", result_class(function)?)); + } + } + for note in document.notes { + render_admonition(writer, "note", note); + } + for warning in document.warnings { + render_admonition(writer, "warning", warning); + } + Ok(()) +} + +fn render_functions_index(writer: &mut Writer, functions: &[(&str, &CompiledFunction)]) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: C++ function index"); + }); + writer.write("# C++ function index\n\n| Function | Summary | Module |\n| --- | --- | --- |\n"); + for (slug, function) in functions { + let qualified = format!("ptfkit::{slug}::{}", function.core.name); + writer.line(format_args!( + "| [`{qualified}`](modules/{slug}.md#{}) | {} | [`ptfkit.{slug}`](modules/{slug}.md) |", + function_anchor(&function.core.name), + escape_table(docs::for_function(spec(function)).summary), + )); + } +} + +fn signature(function: &CompiledFunction) -> Result { + let result = match &function.core.output { + Output::Scalar => "double".to_owned(), + Output::Struct(_) => result_class(function)?.to_owned(), + }; + Ok(format!( + "[[nodiscard]]\ninline {result} {}({})", + function.core.name, + function + .core + .inputs + .iter() + .map(|input| format!("double {input}")) + .collect::>() + .join(", ") + )) +} + +fn spec(function: &CompiledFunction) -> &crate::model::Function { + &function.entry.spec.functions[function.function_index] +} + +fn result_class(function: &CompiledFunction) -> Result<&str> { + spec(function) + .result_class() + .ok_or_else(|| anyhow!("record output has no result class")) +} + +fn function_anchor(name: &str) -> String { + format!("function-{name}") +} + +fn parameter_details(parameter: &Parameter) -> String { + format!( + "{} ({})", + escape_table(¶meter.description), + escape_table(¶meter.unit) + ) +} + +fn render_admonition(writer: &mut Writer, kind: &str, body: &str) { + markdown::admonition(writer, kind, body, escape_text); +} + +fn escape_text(value: &str) -> String { + value.replace('\\', "\\\\").replace('`', "\\`") +} + +fn escape_table(value: &str) -> String { + escape_text(value).replace('\n', " ").replace('|', "\\|") +} + +fn natural_sort_key(value: &str) -> String { + crate::targets::python::natural_sort_key(value) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + + fn rendered_files() -> Vec { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("codegen directory has a repository parent"); + let entries = crate::specs::load(root).expect("repository specifications load"); + let compiled = + crate::compile::functions(entries).expect("repository specifications compile"); + render(&compiled).expect("C++ documentation renders") + } + + fn contents<'a>(files: &'a [GeneratedFile], path: &str) -> &'a str { + &files + .iter() + .find(|file| file.path == Path::new(path)) + .unwrap_or_else(|| panic!("missing generated file {path}")) + .contents + } + + #[test] + fn documents_modules_functions_and_record_fields_from_compiled_sources() { + let files = rendered_files(); + let index = contents(&files, "index.md"); + let rawls = contents(&files, "modules/rawls1982.md"); + let function_index = contents(&files, "functions.md"); + + assert!(index.starts_with( + "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: C++ API reference\n---\n" + )); + assert!(rawls.starts_with( + "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: C++ module ptfkit.rawls1982\nnav-title: ptfkit.rawls1982\n---\n" + )); + assert!(index.contains("[`ptfkit`](modules/ptfkit.md)")); + assert!(rawls.contains("import ptfkit.rawls1982;")); + assert!(rawls.contains("**Exported namespace:** `ptfkit::rawls1982`")); + assert!(rawls.contains("## `Rawls1982PTFResult`")); + assert!(rawls.contains("| `theta_4` | Volumetric water content at -4 kPa. (cm^3/cm^3) |")); + assert!(rawls.contains("[[nodiscard]]\ninline double calc_ptf_rawls1982_theta_1500")); + assert!(rawls.contains("inline Rawls1982PTFResult calc_ptf_rawls1982_full_wrc")); + assert!(rawls.contains("{#function-calc_ptf_rawls1982_theta_1500}")); + assert!(function_index.contains( + "[`ptfkit::rawls1982::calc_ptf_rawls1982_theta_1500`](modules/rawls1982.md#function-calc_ptf_rawls1982_theta_1500)" + )); + assert!(rawls.contains("[PTF catalog page](../../../ptf-catalog/sources/rawls1982.md)")); + } + + #[test] + fn umbrella_lists_each_source_module_once_in_natural_order() { + let files = rendered_files(); + let umbrella = contents(&files, "modules/ptfkit.md"); + let index = contents(&files, "index.md"); + let source_modules = files + .iter() + .filter(|file| { + file.path.starts_with("modules") && file.path != Path::new("modules/ptfkit.md") + }) + .count(); + + assert_eq!(umbrella.matches("- [`ptfkit.").count(), source_modules); + assert!(index.find("[`ptfkit`]( ").is_none()); + assert!( + index.find("[`ptfkit`](modules/ptfkit.md)").unwrap() + < index.find("ptfkit.ahuja1984").unwrap() + ); + assert!( + umbrella.find("ptfkit.ahuja1984").unwrap() + < umbrella.find("ptfkit.aimrun2009").unwrap() + ); + } + + #[test] + fn escapes_markdown_sensitive_text() { + assert_eq!(escape_text("a `name` \\ value"), "a \\`name\\` \\\\ value"); + assert_eq!(escape_table("a|b\nc"), "a\\|b c"); + } + + #[test] + fn function_anchors_are_stable() { + assert_eq!( + function_anchor("calc_ptf_rawls1982"), + "function-calc_ptf_rawls1982" + ); + } + + #[test] + fn natural_ordering_handles_numeric_suffixes() { + assert!(natural_sort_key("calc_2") < natural_sort_key("calc_10")); + } +} diff --git a/codegen/src/targets/reference/mod.rs b/codegen/src/targets/reference/mod.rs new file mode 100644 index 0000000..b3fd5df --- /dev/null +++ b/codegen/src/targets/reference/mod.rs @@ -0,0 +1,5 @@ +//! Language API-reference documentation products. + +pub(super) mod c; +pub(super) mod cpp; +pub(super) mod python; diff --git a/codegen/src/targets/python_documentation.rs b/codegen/src/targets/reference/python.rs similarity index 59% rename from codegen/src/targets/python_documentation.rs rename to codegen/src/targets/reference/python.rs index 3d2408c..7119ff2 100644 --- a/codegen/src/targets/python_documentation.rs +++ b/codegen/src/targets/reference/python.rs @@ -1,47 +1,70 @@ -use std::path::PathBuf; - -use crate::model::Entry; - -use super::{GeneratedFile, documentation::FRONTMATTER_HEADER, py::natural_sort_key}; - -pub(super) fn render(entries: &[Entry]) -> Vec { +use crate::{ + model::Entry, + output::GeneratedFile, + render::{Render, Writer, markdown}, + targets::python::natural_sort_key, +}; + +pub(crate) fn render(entries: &[Entry]) -> Vec { let mut entries = entries.iter().collect::>(); entries.sort_by_key(|entry| natural_sort_key(&entry.slug)); - let mut files = vec![file("index.md", index(&entries))]; + let mut files = vec![markdown::markdown_file("index.md", |writer| { + IndexPage { entries: &entries }.render(writer); + })]; for entry in entries { - files.push(page(&entry.slug)); + files.push(markdown::markdown_file( + format!("{}.md", entry.slug), + |writer| ModulePage { entry }.render(writer), + )); } files } -fn file(path: impl Into, contents: String) -> GeneratedFile { - GeneratedFile::new(path.into(), format!("{}\n", contents.trim_end())) +struct IndexPage<'a> { + entries: &'a [&'a Entry], } -fn index(entries: &[&Entry]) -> String { - let mut text = format!("---\n{FRONTMATTER_HEADER}title: Python API reference\n---\n\n"); - text.push_str("# Python API reference\n\n"); - text.push_str("ptfkit's Python API is organized around public source modules.\n\n"); - text.push_str("## Modules\n\n"); - for entry in entries { - text.push_str(&format!( - "- [`ptfkit.{slug}`]({slug}.md) — {summary}\n", - slug = entry.slug, - summary = entry.spec.source.summary - )); +impl Render for IndexPage<'_> { + fn render(&self, writer: &mut Writer) { + markdown::generated_frontmatter(writer, |writer| { + writer.line("title: Python API reference"); + }); + writer.write( + "# Python API reference\n\nptfkit's Python API is organized around public source modules.\n\n## Modules\n\n", + ); + for entry in self.entries { + ModuleReference { entry }.render(writer); + } } - text } -fn page(slug: &str) -> GeneratedFile { - let module = format!("ptfkit.{slug}"); - file( - format!("{slug}.md"), - format!( - "---\n{FRONTMATTER_HEADER}title: Python module {module}\nnav-title: {module}\n---\n\n::: {module}" - ), - ) +struct ModulePage<'a> { + entry: &'a Entry, +} + +impl Render for ModulePage<'_> { + fn render(&self, writer: &mut Writer) { + let slug = &self.entry.slug; + let module = format!("ptfkit.{slug}"); + markdown::generated_frontmatter(writer, |writer| { + writer.line(format_args!("title: Python module {module}")); + writer.line(format_args!("nav-title: {module}")); + }); + writer.line(format_args!("::: {module}")); + } +} + +struct ModuleReference<'a> { + entry: &'a Entry, +} + +impl Render for ModuleReference<'_> { + fn render(&self, writer: &mut Writer) { + let slug = &self.entry.slug; + let summary = &self.entry.spec.source.summary; + writer.line(format_args!("- [`ptfkit.{slug}`]({slug}.md) — {summary}")); + } } #[cfg(test)] @@ -88,7 +111,7 @@ mod tests { let page = contents(&files, &format!("{}.md", entry.slug)); assert!(page.starts_with(&format!( - "---\n{FRONTMATTER_HEADER}title: Python module {module}\nnav-title: {module}\n---\n" + "---\n# @generated by ptfkit-codegen; DO NOT EDIT.\n\ntitle: Python module {module}\nnav-title: {module}\n---\n" ))); assert!(page.contains(&format!("::: {module}"))); } diff --git a/codegen/src/targets/rs.rs b/codegen/src/targets/rust.rs similarity index 89% rename from codegen/src/targets/rs.rs rename to codegen/src/targets/rust.rs index 85add22..13a77ac 100644 --- a/codegen/src/targets/rs.rs +++ b/codegen/src/targets/rust.rs @@ -5,22 +5,26 @@ use proc_macro2::{Ident, Literal, TokenStream, TokenTree}; use quote::{format_ident, quote}; use crate::{ - model::{CompiledFunction, Function, Output, Parameter, Scope, Source}, + documentation::{self as docs, FunctionDocument, Returns, SourceDocument}, + model::{CompiledFunction, Output}, semantic::{self, BinaryOp, Expr, MathFunction, Number, Reference, UnaryOp}, }; -use super::{documentation, group_by_source}; +use crate::{output::GeneratedFile, targets::group_by_source}; pub(super) const HEADER: &str = "// @generated by ptfkit-codegen; DO NOT EDIT.\n"; -pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { +pub(crate) fn render(functions: &[CompiledFunction]) -> Result> { group_by_source(functions) .into_iter() .map(|(slug, functions)| { let first = functions .first() .expect("generated source contains at least one function"); - let module_docs = module_doc_tokens(&first.entry.spec.source, &first.entry.spec.scope); + let module_docs = module_doc_tokens(docs::for_source( + &first.entry.spec.source, + &first.entry.spec.scope, + )); let unique_test_modules = functions.len() > 1; let mut defined_result_classes = BTreeSet::new(); let definitions = functions @@ -33,7 +37,7 @@ pub(crate) fn render(functions: &[CompiledFunction]) -> Result>>()?; - Ok(( + Ok(GeneratedFile::new( PathBuf::from(format!("{slug}.rs")), render_tokens(module_docs, quote!(#(#definitions)*)), )) @@ -49,7 +53,7 @@ fn module_tokens( ) -> Result { let function = &resolved.core; let specification = &resolved.entry.spec.functions[resolved.function_index]; - let function_docs = function_doc_tokens(specification); + let function_docs = function_doc_tokens(docs::for_function(specification)); let name = format_ident!("{}", function.name); let inputs = function .inputs @@ -157,7 +161,7 @@ fn output_tokens( .iter() .find(|parameter| field == parameter.name) .expect("core output field matches specification"); - let docs = doc_tokens([documentation::parameter_details(parameter)]); + let docs = doc_tokens([docs::parameter_details(parameter)]); quote!(#docs pub #field: f64) }); let values = fields.iter().map(|field| format_ident!("{field}")); @@ -177,75 +181,65 @@ fn output_tokens( } } -fn module_doc_tokens(source: &Source, scope: &Scope) -> TokenStream { +fn module_doc_tokens(document: SourceDocument<'_>) -> TokenStream { let mut lines = vec![ - source.summary.clone(), + document.summary.into(), String::new(), "# Reference".into(), String::new(), - source.citation_apa.clone(), + document.reference.citation.into(), ]; - if let Some(doi) = &source.doi { + if let Some(doi) = document.reference.doi { lines.push(format!("DOI: {} ({})", doi.identifier, doi.url)); } - if let Some(territory) = &scope.territory { + if let Some(territory) = document.territory { lines.extend([ String::new(), "# Territory".into(), String::new(), - territory.clone(), + territory.into(), ]); } - if let Some(dataset) = &scope.dataset { + if let Some(dataset) = document.dataset { lines.extend([ String::new(), "# Dataset".into(), String::new(), - dataset.clone(), + dataset.into(), ]); } inner_doc_tokens(lines) } -fn function_doc_tokens(function: &Function) -> TokenStream { +fn function_doc_tokens(document: FunctionDocument<'_>) -> TokenStream { let mut lines = vec![ - function.public_api.summary.clone(), + document.summary.into(), String::new(), "# Arguments".into(), String::new(), ]; lines.extend( - function - .inputs + document + .parameters .iter() - .map(|parameter| format!("* {}", documentation::parameter_documentation(parameter))), + .map(|parameter| format!("* {}", docs::parameter_documentation(parameter))), ); lines.extend([String::new(), "# Returns".into(), String::new()]); - lines.extend(return_doc_lines( - function.result_class(), - function.outputs.fields(), - )); - if let Some(territory) = &function.scope.territory { + lines.extend(return_doc_lines(document.returns)); + if let Some(territory) = document.territory { lines.extend([ String::new(), "# Territory".into(), String::new(), - territory.clone(), + territory.into(), ]); } let models = [ - function - .scope + document .models .h_theta - .as_ref() .map(|model| format!("h(theta): {model}")), - function - .scope - .models - .k_h - .as_ref() - .map(|model| format!("k(h): {model}")), + document.models.k_h.map(|model| format!("k(h): {model}")), ] .into_iter() .flatten() @@ -258,22 +252,28 @@ fn function_doc_tokens(function: &Function) -> TokenStream { String::new(), "# Notes".into(), String::new(), - format!("Prediction target: {}", function.scope.prediction_target), + format!("Prediction target: {}", document.remarks.prediction_target), ]); - lines.extend(function.documentation.notes.iter().cloned()); - if !function.documentation.warnings.is_empty() { + lines.extend(document.notes.iter().cloned()); + if !document.warnings.is_empty() { lines.extend([String::new(), "# Warnings".into(), String::new()]); - lines.extend(function.documentation.warnings.iter().cloned()); + lines.extend(document.warnings.iter().cloned()); } doc_tokens(lines) } -fn return_doc_lines(result_class: Option<&str>, outputs: &[Parameter]) -> Vec { - match result_class { - Some(result_class) => vec![format!("A [`{result_class}`].")], - None => outputs +fn return_doc_lines(returns: Returns<'_>) -> Vec { + match returns { + Returns::Record { name, fields } => { + debug_assert!( + !fields.is_empty(), + "record outputs contain at least one field" + ); + vec![format!("A [`{name}`].")] + } + Returns::Scalar(output) => std::slice::from_ref(output) .iter() - .map(|parameter| format!("* {}", documentation::parameter_documentation(parameter))) + .map(|parameter| format!("* {}", docs::parameter_documentation(parameter))) .collect(), } } @@ -570,6 +570,8 @@ fn render_tokens(module_docs: TokenStream, tokens: TokenStream) -> String { #[cfg(test)] mod tests { use super::*; + use crate::documentation::Returns; + use crate::model::Parameter; use crate::semantic::{BinaryOp, Expr, MathFunction, Reference}; fn number(value: f64) -> Expr { @@ -715,11 +717,11 @@ mod tests { }; assert_eq!( - documentation::parameter_documentation(¶meter), + docs::parameter_documentation(¶meter), "theta_s: Saturated water content. (cm^3/cm^3)" ); assert_eq!( - documentation::parameter_details(¶meter), + docs::parameter_details(¶meter), "Saturated water content. (cm^3/cm^3)" ); } @@ -727,7 +729,15 @@ mod tests { #[test] fn record_return_documentation_links_to_the_result_type() { assert_eq!( - return_doc_lines(Some("Li2007PTFResult"), &[]), + return_doc_lines(Returns::Record { + name: "Li2007PTFResult", + fields: &[Parameter { + name: "theta_s".into(), + unit: "cm^3/cm^3".into(), + domain: None, + description: "Saturated water content.".into(), + }], + }), ["A [`Li2007PTFResult`]."] ); } diff --git a/docs/src/contributing/development.md b/docs/src/contributing/development.md index 6f6df4e..dc04efa 100644 --- a/docs/src/contributing/development.md +++ b/docs/src/contributing/development.md @@ -56,6 +56,15 @@ just generate A second generation run must leave the working tree unchanged. +To check this across every codegen-owned target family, run: + +```sh +just check-generated +``` + +The command regenerates the targets through the normal pipeline and reports +added, removed, or modified generated files. + ### Generation conventions The specification filename stem is the APA-style source slug. Codegen uses it diff --git a/targets/ptfkit-native/cmake/ptfkitModules.cmake b/targets/ptfkit-native/cmake/ptfkitModules.cmake index 7fe39fa..112d3d5 100644 --- a/targets/ptfkit-native/cmake/ptfkitModules.cmake +++ b/targets/ptfkit-native/cmake/ptfkitModules.cmake @@ -1,4 +1,5 @@ # @generated by ptfkit-codegen; DO NOT EDIT. + set(PTFKIT_CPP_MODULES "${CMAKE_CURRENT_LIST_DIR}/../cpp/ptfkit.cppm" "${CMAKE_CURRENT_LIST_DIR}/../cpp/ahuja1984.cppm"