From a7a438eba06974ca24dfcdfd0c1e547abd6254ee Mon Sep 17 00:00:00 2001 From: Boris Pelakh Date: Thu, 12 Mar 2026 13:07:09 -0400 Subject: [PATCH] Update code and documentation for new name. Add README. --- .github/workflows/tag.yml | 2 +- Cargo.lock | 4 +- Cargo.toml | 4 +- README.md | 58 +++++++++++ src/lib.rs | 10 +- src/main.rs | 6 +- tests/fixtures/escaped_chars.csv | 5 + tests/fixtures/escaped_chars.rq | 9 ++ tests/main.rs | 163 +++++++++++++++++++++++++++---- 9 files changed, 227 insertions(+), 34 deletions(-) create mode 100644 README.md create mode 100644 tests/fixtures/escaped_chars.csv create mode 100644 tests/fixtures/escaped_chars.rq diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 04552f3..cb92b01 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -102,7 +102,7 @@ jobs: shell: bash run: | # Replace with the name of your binary - binary_name="oxi_tarql" + binary_name="oxi_gen" dirname="$binary_name-${{ env.RELEASE_VERSION }}-${{ matrix.target }}" mkdir "$dirname" diff --git a/Cargo.lock b/Cargo.lock index d362269..a45472a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -339,8 +339,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] -name = "oxi_tarql" -version = "0.3.1" +name = "oxi_gen" +version = "0.4.0" dependencies = [ "async-channel", "clap", diff --git a/Cargo.toml b/Cargo.toml index 8d28f0f..6022d71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "oxi_tarql" -version = "0.3.1" +name = "oxi_gen" +version = "0.4.0" edition = "2024" [dependencies] diff --git a/README.md b/README.md new file mode 100644 index 0000000..3a4cff4 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# oxi-gen + +A high-performance command-line tool for converting CSV/TSV files to RDF (Turtle or N-Triples) using SPARQL CONSTRUCT queries. Inspired by [Tarql](https://tarql.github.io/), oxi-gen is built in Rust on top of the [Oxigraph](https://github.com/oxigraph/oxigraph) stack and leverages multi-threaded processing to handle large datasets efficiently. + +Each row of the input CSV is bound as SPARQL variable substitutions and evaluated against a CONSTRUCT query, producing RDF output. Column headers become variable names (e.g., a `name` column is available as `?name`), and the special variable `?ROWNUM` holds the current row index. + +## Command-Line Options + +``` +oxi_gen -q [OPTIONS] +``` + +| Option | Short | Description | +|---|---|---| +| `--query ` | `-q` | SPARQL CONSTRUCT query file to apply (required) | +| `--input ` | `-i` | Input CSV file. Omit to read from STDIN | +| `--output ` | `-o` | Output file. Omit to write to STDOUT | +| `--delimiter ` | `-d` | CSV delimiter character (default: `,`) | +| `--tab` | `-t` | Treat input as tab-separated (TSV) | +| `--no-header-row` | `-H` | Input has no header row; columns are named `a`–`z`, `A`–`Z` | +| `--normalize` | `-n` | Normalize column names to UPPERCASE | +| `--escape_char ` | `-p` | Escape character (default: `\`) | +| `--quote_char ` | | Quote character (default: `"`) | +| `--ntriples` | | Output N-Triples instead of Turtle | +| `--gzip` | `-g` | Gzip the output (requires `--output`) | +| `--dedup[=N]` | | Deduplicate triples within a sliding window (default window: 1000, range: 1000–5000000) | +| `--test[=N]` | | Process only the first N rows for testing (default: 5, max: 49) | +| `--split ` | | Split column ORIGINAL on DELIMITER, binding each value to SPLIT. Can be repeated | +| `--bind-empty-strings` | | Bind empty CSV values as empty string literals instead of skipping them | + +## Custom SPARQL Functions + +oxi-gen registers two custom functions under the `tarql:` prefix (`https://semanticarts.com/tarql/`): + +- **`tarql:expandPrefix(?prefix)`** — returns the IRI for a given prefix name declared in the query. +- **`tarql:expandPrefixedName(?qname)`** — expands a prefixed name (e.g., `"foaf:name"`) into a full IRI node. + +## Building from Source + +### Prerequisites + +- [Rust](https://www.rust-lang.org/tools/install) (1.85+ required for edition 2024) + +### Build a release binary + +```sh +git clone git@github.com:semanticarts/oxi-gen.git +cd oxi-gen +cargo build --release +``` + +The optimized binary will be at `target/release/oxi_gen`. The release profile is configured with LTO, single codegen unit, and abort-on-panic for maximum performance. + +### Run directly with Cargo + +```sh +cargo run --release -- -q query.sparql -i data.csv -o output.ttl +``` diff --git a/src/lib.rs b/src/lib.rs index d6588aa..0706a07 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,7 +23,7 @@ use clap::{Arg, ArgAction, ArgMatches, command, value_parser}; #[allow(dead_code)] #[derive(Default)] -pub struct OxiTarql { +pub struct OxiGen { pub delimiter: String, pub tab: bool, pub test: u32, @@ -41,7 +41,7 @@ pub struct OxiTarql { pub split: Vec<(String, String, String)>, } -impl OxiTarql { +impl OxiGen { pub fn transform(&mut self) -> Result<(), Box> { let num_workers: usize = num_cpus::get(); @@ -424,7 +424,7 @@ where I: IntoIterator, { command!() - .about("Convert CSV file to RDF using SPARQL") + .about("oxi-gen: Convert CSV file to RDF using SPARQL") .arg( Arg::new("delimiter") .short('d') @@ -547,7 +547,7 @@ where .get_matches_from(args) } -pub fn configure_transform(args: I) -> OxiTarql +pub fn configure_transform(args: I) -> OxiGen where I: IntoIterator, { @@ -565,7 +565,7 @@ where } }; - OxiTarql { + OxiGen { delimiter: matches.get_one::("delimiter").unwrap().to_string(), tab: matches.get_flag("tab"), test: match matches.get_one::("test") { diff --git a/src/main.rs b/src/main.rs index 34a329f..70107b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,14 @@ -use oxi_tarql::configure_transform; +use oxi_gen::configure_transform; use std::{env, time::Instant}; fn main() { // parse the supplied arguments let os_args: Vec = env::args_os().map(|a| a.into_string().unwrap()).collect(); - let mut tarql = configure_transform(os_args); + let mut transform = configure_transform(os_args); let start = Instant::now(); - tarql.transform().expect("Transformation failed"); + transform.transform().expect("Transformation failed"); let duration = Instant::now().duration_since(start); eprintln!("Processing complete in {} seconds", duration.as_secs_f32()); diff --git a/tests/fixtures/escaped_chars.csv b/tests/fixtures/escaped_chars.csv new file mode 100644 index 0000000..7216a3a --- /dev/null +++ b/tests/fixtures/escaped_chars.csv @@ -0,0 +1,5 @@ +id,description +1,"C:\\Users\\test\\path" +2,"line1\\nline2\\ttab" +3,"say \"hello world\"" +4,"mixed: C:\\path \"quoted\"" diff --git a/tests/fixtures/escaped_chars.rq b/tests/fixtures/escaped_chars.rq new file mode 100644 index 0000000..ed7ddaf --- /dev/null +++ b/tests/fixtures/escaped_chars.rq @@ -0,0 +1,9 @@ +prefix : +construct { + ?row_iri + :description ?description ; + . +} +where { + BIND(tarql:expandPrefixedName(CONCAT(":row_", STR(?ROWNUM))) AS ?row_iri) +} diff --git a/tests/main.rs b/tests/main.rs index ff16038..3b47ed4 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -1,13 +1,14 @@ use flate2::read::GzDecoder; -use oxi_tarql::configure_transform; +use oxi_gen::configure_transform; use oxrdfio::RdfParser; +use std::collections::HashMap; use std::fs; use std::path::PathBuf; #[test] fn test_integration_split_with_custom_functions() { // Create a temporary file for output - let temp_file = std::env::temp_dir().join("oxi_tarql_test_output.nt"); + let temp_file = std::env::temp_dir().join("oxi_gen_test_output.nt"); // Clean up any existing temp file let _ = std::fs::remove_file(&temp_file); @@ -31,7 +32,7 @@ fn test_integration_split_with_custom_functions() { // Build command-line arguments for configure_transform let args = vec![ - "oxi_tarql".to_string(), + "oxi_gen".to_string(), "--input".to_string(), input_path.to_str().unwrap().to_string(), "--query".to_string(), @@ -50,10 +51,10 @@ fn test_integration_split_with_custom_functions() { " ".to_string(), ]; - let mut tarql = configure_transform(args); + let mut transform = configure_transform(args); // Run the transformation - let result = tarql.transform(); + let result = transform.transform(); assert!( result.is_ok(), "Transform should succeed: {:?}", @@ -87,7 +88,7 @@ fn test_integration_split_with_custom_functions() { #[test] fn test_integration_turtle_serialization() { // Create a temporary file for output - let temp_file = std::env::temp_dir().join("oxi_tarql_test_output.ttl"); + let temp_file = std::env::temp_dir().join("oxi_gen_test_output.ttl"); // Clean up any existing temp file let _ = std::fs::remove_file(&temp_file); @@ -111,7 +112,7 @@ fn test_integration_turtle_serialization() { // Build command-line arguments for configure_transform let args = vec![ - "oxi_tarql".to_string(), + "oxi_gen".to_string(), "--input".to_string(), input_path.to_str().unwrap().to_string(), "--query".to_string(), @@ -130,10 +131,10 @@ fn test_integration_turtle_serialization() { " ".to_string(), ]; - let mut tarql = configure_transform(args); + let mut transform = configure_transform(args); // Run the transformation - let result = tarql.transform(); + let result = transform.transform(); assert!( result.is_ok(), "Transform should succeed: {:?}", @@ -160,7 +161,7 @@ fn test_integration_with_dedup_and_gzip() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let input_path = manifest_dir.join("tests/fixtures/data_100.csv"); let query_path = manifest_dir.join("tests/fixtures/with_dup.rq"); - let temp_file = std::env::temp_dir().join("oxi_tarql_test_dedup.nt.gz"); + let temp_file = std::env::temp_dir().join("oxi_gen_test_dedup.nt.gz"); // Clean up any existing temp file let _ = std::fs::remove_file(&temp_file); @@ -179,7 +180,7 @@ fn test_integration_with_dedup_and_gzip() { // Build command-line arguments for configure_transform let args = vec![ - "oxi_tarql".to_string(), + "oxi_gen".to_string(), "--input".to_string(), input_path.to_str().unwrap().to_string(), "--query".to_string(), @@ -191,10 +192,10 @@ fn test_integration_with_dedup_and_gzip() { "--dedup=1000".to_string(), ]; - let mut tarql = configure_transform(args); + let mut transform = configure_transform(args); // Run the transformation - let result = tarql.transform(); + let result = transform.transform(); assert!( result.is_ok(), "Transform should succeed: {:?}", @@ -249,7 +250,7 @@ fn test_integration_with_dedup_and_gzip() { #[test] fn test_integration_optional_field_empty_values() { // Create a temporary file for output - let temp_file = std::env::temp_dir().join("oxi_tarql_test_optional.nt"); + let temp_file = std::env::temp_dir().join("oxi_gen_test_optional.nt"); // Clean up any existing temp file let _ = std::fs::remove_file(&temp_file); @@ -273,7 +274,7 @@ fn test_integration_optional_field_empty_values() { // Build command-line arguments WITHOUT --bind-empty-strings (default behavior) let args = vec![ - "oxi_tarql".to_string(), + "oxi_gen".to_string(), "--input".to_string(), input_path.to_str().unwrap().to_string(), "--query".to_string(), @@ -283,10 +284,10 @@ fn test_integration_optional_field_empty_values() { "--ntriples".to_string(), ]; - let mut tarql = configure_transform(args); + let mut transform = configure_transform(args); // Run the transformation - let result = tarql.transform(); + let result = transform.transform(); assert!( result.is_ok(), "Transform should succeed: {:?}", @@ -348,7 +349,7 @@ fn test_integration_optional_field_empty_values() { #[test] fn test_integration_expand_prefixed_name_with_empty_values() { // Create a temporary file for output - let temp_file = std::env::temp_dir().join("oxi_tarql_test_successor.nt"); + let temp_file = std::env::temp_dir().join("oxi_gen_test_successor.nt"); // Clean up any existing temp file let _ = std::fs::remove_file(&temp_file); @@ -372,7 +373,7 @@ fn test_integration_expand_prefixed_name_with_empty_values() { // Build command-line arguments WITHOUT --bind-empty-strings (default behavior) let args = vec![ - "oxi_tarql".to_string(), + "oxi_gen".to_string(), "--input".to_string(), input_path.to_str().unwrap().to_string(), "--query".to_string(), @@ -382,10 +383,10 @@ fn test_integration_expand_prefixed_name_with_empty_values() { "--ntriples".to_string(), ]; - let mut tarql = configure_transform(args); + let mut transform = configure_transform(args); // Run the transformation - let result = tarql.transform(); + let result = transform.transform(); assert!( result.is_ok(), "Transform should succeed: {:?}", @@ -453,3 +454,123 @@ fn test_integration_expand_prefixed_name_with_empty_values() { "Should contain both subject two and successor reference to two" ); } + +#[test] +fn test_integration_escaped_special_characters() { + let temp_file = std::env::temp_dir().join("oxi_gen_test_escaped.nt"); + let _ = std::fs::remove_file(&temp_file); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let input_path = manifest_dir.join("tests/fixtures/escaped_chars.csv"); + let query_path = manifest_dir.join("tests/fixtures/escaped_chars.rq"); + + assert!( + input_path.exists(), + "Input file should exist: {:?}", + input_path + ); + assert!( + query_path.exists(), + "Query file should exist: {:?}", + query_path + ); + + let args = vec![ + "oxi_gen".to_string(), + "--input".to_string(), + input_path.to_str().unwrap().to_string(), + "--query".to_string(), + query_path.to_str().unwrap().to_string(), + "--output".to_string(), + temp_file.to_str().unwrap().to_string(), + "--ntriples".to_string(), + ]; + + let mut transform = configure_transform(args); + let result = transform.transform(); + assert!( + result.is_ok(), + "Transform should succeed: {:?}", + result.err() + ); + + assert!( + temp_file.exists(), + "Output file should exist at {:?}", + temp_file + ); + + // Parse the N-Triples output with a proper RDF parser to validate correctness + let file = fs::File::open(&temp_file).expect("Should open output file"); + let parser = RdfParser::from_format(oxrdfio::RdfFormat::NTriples).for_reader(file); + + let description_pred = "https://test.com/d/description"; + let mut descriptions: HashMap = HashMap::new(); + + for q in parser { + let quad = q.expect("All output triples must be valid N-Triples"); + if quad.predicate.as_str() == description_pred + && let oxrdf::Term::Literal(lit) = &quad.object + { + let subj = quad.subject.to_string(); + descriptions.insert(subj, lit.value().to_string()); + } + } + + let _ = std::fs::remove_file(&temp_file); + + // Should have 4 rows, each producing a description triple + assert_eq!( + descriptions.len(), + 4, + "Expected 4 description literals, got {}", + descriptions.len() + ); + + // Row 1: CSV `\\` escapes produce literal backslashes in the value. + // Verifies that backslash characters survive CSV→SPARQL→RDF serialization→parse round-trip. + let row0 = descriptions + .get("") + .expect("Should have description for row 0"); + assert_eq!( + row0, "C:\\Users\\test\\path", + "Row 0: backslash-escaped path should produce literal backslashes" + ); + + // Row 2: literal `\n` and `\t` sequences (not control characters). + // The CSV escape char consumes the first `\`, so `\\n` → `\n` as two chars. + let row1 = descriptions + .get("") + .expect("Should have description for row 1"); + assert_eq!( + row1, "line1\\nline2\\ttab", + "Row 1: literal backslash-n and backslash-t sequences should be preserved" + ); + // Confirm these are actual backslash + letter, not control characters + assert!( + !row1.contains('\n') && !row1.contains('\t'), + "Row 1 must not contain real newline or tab control characters" + ); + + // Row 3: CSV `\"` escape produces literal quote characters in the value. + let row2 = descriptions + .get("") + .expect("Should have description for row 2"); + assert_eq!( + row2, "say \"hello world\"", + "Row 2: escaped quotes should produce literal double-quote characters" + ); + + // Row 4: both backslash and quote escapes together in one value. + let row3 = descriptions + .get("") + .expect("Should have description for row 3"); + assert_eq!( + row3, "mixed: C:\\path \"quoted\"", + "Row 3: mixed backslash and quote escapes should both be preserved" + ); + assert!( + row3.contains('\\') && row3.contains('"'), + "Row 3 must contain both backslash and quote characters" + ); +}