From ec1b1dc4611d267ab9c97cd18b7e59ae77ef91bc Mon Sep 17 00:00:00 2001 From: DeviPriya-Kolla Date: Sun, 15 Dec 2024 17:13:54 -0600 Subject: [PATCH 1/2] Implemented read_tbe function for reading and parsing TBE files using Rust --- rust/Cargo.lock | 119 +++++++++++++++++++++++++++++++++ rust/Cargo.toml | 10 +++ rust/README.md | 31 ++++++++- rust/src/functions/mod.rs | 1 + rust/src/functions/read_TBE.rs | 90 +++++++++++++++++++++++++ rust/src/main.rs | 29 ++++++++ rust/src/models.rs | 22 ++++++ 7 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/src/functions/mod.rs create mode 100644 rust/src/models.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..2d1f658 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,119 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rust" +version = "0.1.0" +dependencies = [ + "csv", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..ebdbbf1 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rust" +version = "0.1.0" +edition = "2021" + +[dependencies] +csv = "1.1" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" \ No newline at end of file diff --git a/rust/README.md b/rust/README.md index 0231159..2a13b13 100644 --- a/rust/README.md +++ b/rust/README.md @@ -16,7 +16,36 @@ ## read_TBE -(Add your content here) +### Getting Started + +#### Prerequisites + +- **Rust**: Make sure you have [Rust](https://www.rust-lang.org/learn/get-started) installed. +- **Cargo**: Rust's package manager, which is bundled with the Rust installation. +- **Install VS Code extensions**: Rust Extension Pack and rust-analyzer and Cargo +- **Check environment variables**: /home//.cargo/bin (for example) +- Check the installations: + rustc --version and cargo --version + + +### Installation + +1. Clone the repository to your local machine: git clone https://github.com/oss-slu/tbe.git +2. Go to tbe/rust/ +3. Initialize a New Cargo Project - cargo init (if it's not existed) +4. This will create a Cargo.toml file +5. Then , the folder structure should be like + rust/ + ├── Cargo.toml + ├── src/ + │ └── main.rs +6. Ensure you've the necessary dependencies in Cargo.toml +7. Run the project from the rust/ directory using - cargo run +8. When you run it'll build the project - Cargo.lock and target/ directory +9. If you already have Cargo.lock and no target/ dir then use " cargo build " to build the project + + + ## strip_header diff --git a/rust/src/functions/mod.rs b/rust/src/functions/mod.rs new file mode 100644 index 0000000..1ea8e9a --- /dev/null +++ b/rust/src/functions/mod.rs @@ -0,0 +1 @@ +pub mod read_tbe; // Expose the read_TBE module \ No newline at end of file diff --git a/rust/src/functions/read_TBE.rs b/rust/src/functions/read_TBE.rs index e69de29..a3c12f6 100644 --- a/rust/src/functions/read_TBE.rs +++ b/rust/src/functions/read_TBE.rs @@ -0,0 +1,90 @@ +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, BufRead, BufReader}; + +#[derive(Debug, Default)] +pub struct Table { + pub data: Vec>, // Main table data + pub att: HashMap>, // Attribute data linked to table + pub cmt: HashMap>, // Comment data linked to table +} + +pub fn parse_tbe(file_path: &str) -> io::Result> { + let file = File::open(file_path)?; + let reader = BufReader::new(file); + + let mut tables: HashMap = HashMap::new(); + let mut current_table_name = String::new(); + let mut headers: Vec = Vec::new(); + let mut capturing_data = false; + + for line in reader.lines() { + let line = line?; + let line = line.trim(); + + // Handle table section + if line.starts_with("TBL") { + let parts: Vec<&str> = line.split(',').collect(); + if parts.len() > 1 { + current_table_name = parts[0].split_whitespace().nth(1).unwrap_or_default().to_string(); + headers = parts.iter().skip(1).map(|s| s.trim().to_string()).collect(); + + println!("Parsed table: {} with headers: {:?}", current_table_name, headers); // Debug log + tables.entry(current_table_name.clone()).or_insert_with(Table::default); + } + } + // Start reading table data + else if line.starts_with("BGN") { + capturing_data = true; + } + // End of table data + else if line.starts_with("EOT") { + capturing_data = false; + } + // Handle ATT (Attribute) section + else if line.starts_with("ATT") { + let parts: Vec<&str> = line.split(',').collect(); + if parts.len() > 2 { + let att_name = parts[1].trim().to_string(); + let att_values: Vec = parts.iter().skip(2).map(|s| s.trim().to_string()).collect(); + + if let Some(table) = tables.get_mut(¤t_table_name) { + table.att.insert(att_name, att_values); + } + } + } + // Handle CMT (Comment) section + else if line.starts_with("CMT") { + let parts: Vec<&str> = line.split(',').collect(); + if parts.len() > 2 { + let cmt_name = parts[1].trim().to_string(); + let cmt_values: Vec = parts.iter().skip(2).map(|s| s.trim().to_string()).collect(); + + if let Some(table) = tables.get_mut(¤t_table_name) { + table.cmt.insert(cmt_name, cmt_values); + } + } + } + // Parse table rows + else if capturing_data { + let parts: Vec<&str> = line.split(',').collect(); + if parts.len() == headers.len() { + let row_data: HashMap = headers + .iter() + .zip(parts.iter().map(|s| s.trim().to_string())) + .map(|(h, v)| (h.clone(), v)) + .collect(); + + // Push the cloned row into the table and log the captured row + if let Some(table) = tables.get_mut(¤t_table_name) { + table.data.push(row_data.clone()); // Clone to avoid moving + println!("Captured row: {:?}", row_data); // Debug log + } + } else { + eprintln!("Skipping malformed row: {:?}", parts); // Debug log for malformed rows + } + } + } + + Ok(tables) +} \ No newline at end of file diff --git a/rust/src/main.rs b/rust/src/main.rs index e69de29..450a30d 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -0,0 +1,29 @@ +mod functions; // Include the functions module +use functions::read_tbe::parse_tbe; // Import the parse_tbe function + +use std::io; // Import io for Result +use std::path::PathBuf; // Import PathBuf for path management + +mod models; // Import models where the Site struct is defined + + +fn main() -> io::Result<()> { + let file_path = PathBuf::from("../sample_data/saq_bluesky_bgd_20211001_20230430_inv_tbe.csv"); // Path to the sample TBE file + // let file_path = PathBuf::from("../sample_data/saq_bluesky_dku_20210715_20230131_inv_tbe.csv"); // Path to the sample TBE file + // let file_path = PathBuf::from("../sample_data/saq_bluesky_npl_20220830_20230404_inv_tbe.csv"); // Path to the sample TBE file + println!("Parsing TBE file: {}", file_path.display()); // Use .display() to print the PathBuf + + let result = parse_tbe(file_path.to_str().unwrap()); + + match result { + Ok(sites) => { + // Now sites is a vector of `Site` structs + for site in sites { + println!("{:?}", site); // Print each site + } + } + Err(e) => eprintln!("Error parsing file: {}", e), + } + + Ok(()) +} \ No newline at end of file diff --git a/rust/src/models.rs b/rust/src/models.rs new file mode 100644 index 0000000..6019b38 --- /dev/null +++ b/rust/src/models.rs @@ -0,0 +1,22 @@ +// models.rs +// use serde_derive::Deserialize; // Add the import for the macro + +#[allow(dead_code)] +pub struct Site { + pub zone: String, + pub country: String, + pub sitename: String, + pub utc_offset: String, + pub pm25_scale: String, + pub pm25_offset: String, + pub serial_number: String, + pub plocation: String, + pub latitude: String, + pub longitude: String, + pub is_indoors: String, + pub measurements: String, + pub nrecords: String, + pub date_start: String, + pub date_end: String, + pub time: String, +} \ No newline at end of file From 8bae79cdc992465978ca72403881212e20ba37aa Mon Sep 17 00:00:00 2001 From: DeviPriya-Kolla Date: Sun, 15 Dec 2024 17:35:41 -0600 Subject: [PATCH 2/2] Updated readme.md --- rust/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/README.md b/rust/README.md index 2a13b13..bbf1b9f 100644 --- a/rust/README.md +++ b/rust/README.md @@ -40,9 +40,10 @@ ├── src/ │ └── main.rs 6. Ensure you've the necessary dependencies in Cargo.toml -7. Run the project from the rust/ directory using - cargo run -8. When you run it'll build the project - Cargo.lock and target/ directory -9. If you already have Cargo.lock and no target/ dir then use " cargo build " to build the project +7. Change the path to your sample_data/example.csv as required +8. Run the project from the rust/ directory using - cargo run +9. When you run it'll build the project - Cargo.lock and target/ directory +10. If you already have Cargo.lock and no target/ dir then use " cargo build " to build the project