From e723a65cefdc5a8ac58f99123769e8f84a1253ce Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 12 Sep 2023 15:41:08 -0300 Subject: [PATCH 01/48] WIP --- etk-asm/src/asm.rs | 66 ++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index a0555961..f7cefaa9 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -198,20 +198,13 @@ impl From> for RawOp { /// # assert_eq!(output, hex!("58")); /// # Result::<(), Error>::Ok(()) /// ``` -#[derive(Debug)] + pub struct Assembler { /// Assembled ops, ready to be taken. ready: Vec, - /// Ops that cannot be encoded yet. - pending: VecDeque, - - /// Sum of the size of all the ops in `pending`, or `None` if `pending` contains - /// an unsized op. - pending_len: Option, - - /// Total number of `u8` that have been appended to `ready`. - concrete_len: usize, + /// Assembled ops, not yet ready to be taken. + unrolled: Vec, /// Labels, in `pending`, associated with an `AbstractOp::Label`. declared_labels: HashMap>, @@ -221,19 +214,32 @@ pub struct Assembler { /// Labels, in `pending`, that have been referred to (ex. with push) but /// have not been declared with an `AbstractOp::Label`. - undeclared_labels: HashSet, + undeclared_labels: Vec, + + /// Macros in pending + undeclared_macros: Vec, +} +#[derive(Debug)] +struct PendingMacro { + position: usize, + name: String, + arguments: Vec, +} +#[derive(Debug)] +struct PendingLabel { + position: usize, + name: String, } impl Default for Assembler { fn default() -> Self { Self { ready: Default::default(), - pending: Default::default(), - pending_len: Some(0), - concrete_len: 0, + unrolled: Default::default(), declared_labels: Default::default(), declared_macros: Default::default(), undeclared_labels: Default::default(), + undeclared_macros: Default::default(), } } } @@ -311,8 +317,7 @@ impl Assembler { Ok(self.ready.len()) } - /// Insert explicilty declared macros and labels, via `AbstractOp`, and implictly declared - /// macros and labels via usage in `Op`. + /// Insert explicilty declared macros and labels, via `AbstractOp`. fn declare_content(&mut self, rop: &RawOp) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(ref label)) => { @@ -321,8 +326,15 @@ impl Assembler { return error::DuplicateLabel { label }.fail(); } hash_map::Entry::Vacant(v) => { - v.insert(None); - self.undeclared_labels.remove(label); + v.insert(Some(self.ready.len())); + + for ul in self.undeclared_labels.iter() { + if ul.name == *label { + self.unrolled.insert(ul.position, rop.clone()); + } + } + + self.undeclared_labels.retain(|ul| ul.name != *label); } } } @@ -333,22 +345,20 @@ impl Assembler { } hash_map::Entry::Vacant(v) => { v.insert(defn.to_owned()); + + for um in self.undeclared_macros.iter() { + if um.name == *defn.name() { + self.unrolled.insert(um.position, rop.clone()); + } + } + + self.undeclared_macros.retain(|um| um.name != *defn.name()); } } } _ => (), }; - // Get all labels used by `rop`, check if they've been defined, and if not, note them as - // "undeclared". - if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { - for label in labels { - if !self.declared_labels.contains_key(&label) { - self.undeclared_labels.insert(label.to_owned()); - } - } - } - Ok(()) } From d8da0527e949af1bbf6719468a6dc5a5c0cad279 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 12 Sep 2023 19:29:36 -0300 Subject: [PATCH 02/48] Trying some ideas --- etk-asm/src/asm.rs | 57 +++++++++++++--------------------------------- 1 file changed, 16 insertions(+), 41 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index f7cefaa9..fc904747 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -198,14 +198,11 @@ impl From> for RawOp { /// # assert_eq!(output, hex!("58")); /// # Result::<(), Error>::Ok(()) /// ``` - +#[derive(Debug)] pub struct Assembler { /// Assembled ops, ready to be taken. ready: Vec, - /// Assembled ops, not yet ready to be taken. - unrolled: Vec, - /// Labels, in `pending`, associated with an `AbstractOp::Label`. declared_labels: HashMap>, @@ -214,32 +211,16 @@ pub struct Assembler { /// Labels, in `pending`, that have been referred to (ex. with push) but /// have not been declared with an `AbstractOp::Label`. - undeclared_labels: Vec, - - /// Macros in pending - undeclared_macros: Vec, -} -#[derive(Debug)] -struct PendingMacro { - position: usize, - name: String, - arguments: Vec, -} -#[derive(Debug)] -struct PendingLabel { - position: usize, - name: String, + undeclared_labels: HashSet, } impl Default for Assembler { fn default() -> Self { Self { ready: Default::default(), - unrolled: Default::default(), declared_labels: Default::default(), declared_macros: Default::default(), undeclared_labels: Default::default(), - undeclared_macros: Default::default(), } } } @@ -317,7 +298,8 @@ impl Assembler { Ok(self.ready.len()) } - /// Insert explicilty declared macros and labels, via `AbstractOp`. + /// Insert explicilty declared macros and labels, via `AbstractOp`, and implictly declared + /// macros and labels via usage in `Op`. fn declare_content(&mut self, rop: &RawOp) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(ref label)) => { @@ -326,15 +308,8 @@ impl Assembler { return error::DuplicateLabel { label }.fail(); } hash_map::Entry::Vacant(v) => { - v.insert(Some(self.ready.len())); - - for ul in self.undeclared_labels.iter() { - if ul.name == *label { - self.unrolled.insert(ul.position, rop.clone()); - } - } - - self.undeclared_labels.retain(|ul| ul.name != *label); + v.insert(None); + self.undeclared_labels.remove(label); } } } @@ -345,20 +320,22 @@ impl Assembler { } hash_map::Entry::Vacant(v) => { v.insert(defn.to_owned()); - - for um in self.undeclared_macros.iter() { - if um.name == *defn.name() { - self.unrolled.insert(um.position, rop.clone()); - } - } - - self.undeclared_macros.retain(|um| um.name != *defn.name()); } } } _ => (), }; + // Get all labels used by `rop`, check if they've been defined, and if not, note them as + // "undeclared". + if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { + for label in labels { + if !self.declared_labels.contains_key(&label) { + self.undeclared_labels.insert(label.to_owned()); + } + } + } + Ok(()) } @@ -699,8 +676,6 @@ impl Assembler { Ok(Some(self.push_all(m.contents)?)) } _ => { - assert_eq!(self.pending_len, Some(0)); - self.pending_len = None; self.pending.push_back(RawOp::Op(AbstractOp::Macro( ops::InstructionMacroInvocation { name: name.to_string(), From 027154c807e28356900b77de3893cc54ab20a35e Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Wed, 13 Sep 2023 16:32:34 -0300 Subject: [PATCH 03/48] Still WIP. Need to call finish() at the end. --- etk-asm/src/asm.rs | 394 +++++++++------------------- etk-asm/tests/asm.rs | 30 +++ etk-asm/tests/asm/every-op/test.etk | 7 + 3 files changed, 157 insertions(+), 274 deletions(-) create mode 100644 etk-asm/tests/asm/every-op/test.etk diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index fc904747..49e93428 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -130,9 +130,10 @@ mod error { } pub use self::error::Error; +use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; use crate::ops::expression::{self, Terminal}; use crate::ops::{self, AbstractOp, Assemble, Expression, Imm, MacroDefinition}; -use etk_ops::cancun::Op; +use etk_ops::cancun::{Op, Push1}; use rand::Rng; use snafu::OptionExt; use std::collections::{hash_map, HashMap, HashSet, VecDeque}; @@ -201,7 +202,10 @@ impl From> for RawOp { #[derive(Debug)] pub struct Assembler { /// Assembled ops, ready to be taken. - ready: Vec, + ready: Vec, + + /// Number ob bytes in 'ready' that are ready to be taken. + concrete_len: usize, /// Labels, in `pending`, associated with an `AbstractOp::Label`. declared_labels: HashMap>, @@ -211,16 +215,42 @@ pub struct Assembler { /// Labels, in `pending`, that have been referred to (ex. with push) but /// have not been declared with an `AbstractOp::Label`. - undeclared_labels: HashSet, + undefined_labels: Vec, + + /// Macros, in `pending`, that have been referred to but + /// have not been declared with an `AbstractOp::Macro`. + undefined_macros: Vec, +} + +/// TODO Temporal DOC +#[derive(Debug)] +pub struct PendingLabel { + /// The name of the label. + label: String, + + /// The offset of the label. + offset: usize, +} + +/// TODO Temporal DOC +#[derive(Debug)] +pub struct PendingMacro { + /// The name of the macro. + name: String, + + /// The offset of the macro. + offset: usize, } impl Default for Assembler { fn default() -> Self { Self { ready: Default::default(), + concrete_len: 0, declared_labels: Default::default(), declared_macros: Default::default(), - undeclared_labels: Default::default(), + undefined_labels: Default::default(), + undefined_macros: Default::default(), } } } @@ -231,56 +261,53 @@ impl Assembler { Self::default() } - /// Indicate that the input sequence is complete. Returns any errors that - /// may remain. - pub fn finish(self) -> Result<(), Error> { - if let Some(undef) = self.pending.front() { - return match undef { - RawOp::Op(AbstractOp::Macro(invc)) => error::UndeclaredInstructionMacro { - name: invc.name.clone(), - } - .fail(), - RawOp::Op(op) => { - match op - .clone() - .concretize((&self.declared_labels, &self.declared_macros).into()) - { - Ok(_) => unreachable!(), - Err(ops::Error::ContextIncomplete { - source: expression::Error::UnknownMacro { name, .. }, - .. - }) => error::UndeclaredExpressionMacro { name }.fail(), - Err(ops::Error::ContextIncomplete { - source: expression::Error::UnknownLabel { .. }, - .. - }) => { - let labels = op.expr().unwrap().labels(&self.declared_macros).unwrap(); - let declared = self.declared_labels.into_keys().collect(); - let invoked: HashSet<_> = labels.into_iter().collect(); - let missing = invoked - .difference(&declared) - .cloned() - .collect::>(); - error::UndeclaredLabels { labels: missing }.fail() - } - _ => unreachable!(), - } - } - // bug: if a variable is used when it isn't available, e.g. push1 $size - _ => unreachable!(), - }; + /// Collect any assembled instructions that are ready to be output. + pub fn take(&mut self) -> Vec { + let output = self.finish(); + match output { + Ok(v) => std::mem::take(&mut v.clone()), + Err(e) => { + eprintln!("error: {}", e); + Vec::new() + } } + } - if !self.ready.is_empty() { - panic!("not all assembled bytecode has been taken"); + /// Temporal documentation. TODO + pub fn finish(&mut self) -> Result, Error> { + if !self.undefined_labels.is_empty() { + return error::UndeclaredLabels { + labels: self + .undefined_labels + .iter() + .map(|l| l.label.to_owned()) + .collect::>(), + } + .fail(); } - Ok(()) - } + if !self.undefined_macros.is_empty() { + return error::UndeclaredInstructionMacro { + name: self.undefined_macros[0].name.to_owned(), + } + .fail(); + } - /// Collect any assembled instructions that are ready to be output. - pub fn take(&mut self) -> Vec { - std::mem::take(&mut self.ready) + // Concretize every RawOp ready, and collect the results. + let mut output = Vec::new(); + for ready_op in self.ready.iter() { + if let &RawOp::Op(ref op) = ready_op { + match op + .clone() + .concretize((&self.declared_labels, &self.declared_macros).into()) + { + Ok(cop) => cop.assemble(&mut output), + Err(e) => unreachable!("all ops should be concretizable: {}", e), + } + } + } + + Ok(output) } /// Feed instructions into the `Assembler`. @@ -309,7 +336,6 @@ impl Assembler { } hash_map::Entry::Vacant(v) => { v.insert(None); - self.undeclared_labels.remove(label); } } } @@ -320,21 +346,29 @@ impl Assembler { } hash_map::Entry::Vacant(v) => { v.insert(defn.to_owned()); + + for m in self.undefined_macros.iter() { + if m.name == *defn.name() { + self.ready.insert(m.offset, rop.clone()); + } + } } } } - _ => (), + _ => { + println!("declare_content (salio): {:?}", rop) + } }; // Get all labels used by `rop`, check if they've been defined, and if not, note them as // "undeclared". - if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { - for label in labels { - if !self.declared_labels.contains_key(&label) { - self.undeclared_labels.insert(label.to_owned()); - } - } - } + //if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { + // for label in labels { + // if !self.declared_labels.contains_key(&label) { + // self.undefined_labels.insert(label.to_owned()); + // } + // } + //} Ok(()) } @@ -358,18 +392,10 @@ impl Assembler { return Ok(self.ready.len()); } - self.push_unchecked(rop)?; + self.push_ready(rop)?; Ok(self.ready.len()) } - fn push_unchecked(&mut self, rop: RawOp) -> Result<(), Error> { - if self.pending.is_empty() && self.pending_len.is_some() { - self.push_ready(rop) - } else { - self.push_pending(rop) - } - } - fn push_ready(&mut self, rop: RawOp) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(label)) => { @@ -381,18 +407,7 @@ impl Assembler { Ok(()) } RawOp::Op(AbstractOp::MacroDefinition(_)) => Ok(()), - RawOp::Op(AbstractOp::Macro(ref m)) => { - match self.declared_macros.get(&m.name) { - // Do nothing if the instruction macro has been defined. - Some(MacroDefinition::Instruction(_)) => (), - _ => { - assert_eq!(self.pending_len, Some(0)); - self.pending_len = None; - self.pending.push_back(rop); - } - } - Ok(()) - } + RawOp::Op(AbstractOp::Macro(ref m)) => Ok(()), RawOp::Op(ref op) => { match op .clone() @@ -400,7 +415,7 @@ impl Assembler { { Ok(cop) => { self.concrete_len += cop.size(); - cop.assemble(&mut self.ready); + self.ready.push(rop); } Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { return error::ExpressionTooLarge { @@ -417,205 +432,35 @@ impl Assembler { } .fail() } - Err(_) => { - assert_eq!(self.pending_len, Some(0)); - self.pending_len = rop.size(); - self.pending.push_back(rop); - } + Err(ops::Error::ContextIncomplete { source }) => match source { + UnknownLabel { label, .. } => { + self.ready.push(rop); + self.undefined_labels.push(PendingLabel { + label: label.to_owned(), + offset: self.concrete_len, + }); + } + UnknownMacro { name, .. } => { + self.ready.push(rop); + self.undefined_macros.push(PendingMacro { + name: name.to_owned(), + offset: self.concrete_len, + }); + } + UndefinedVariable { name, .. } => todo!(), + }, } Ok(()) } RawOp::Raw(raw) => { self.concrete_len += raw.len(); - self.ready.extend(raw); + self.ready.push(RawOp::Raw(raw)); Ok(()) } } } - fn push_pending(&mut self, rop: RawOp) -> Result<(), Error> { - // Update total size of pending ops. - if let Some(ref mut pending_len) = self.pending_len { - match rop.size() { - Some(size) => *pending_len += size, - None => self.pending_len = None, - } - } - - // Handle new label and macro definitions. - match (self.pending_len, rop) { - (Some(pending_len), RawOp::Op(AbstractOp::Label(lbl))) => { - // The label has a defined address. - let address = self.concrete_len + pending_len; - let item = self.declared_labels.get_mut(&*lbl).unwrap(); - *item = Some(address); - } - (None, rop @ RawOp::Op(AbstractOp::Label(_))) => { - self.pending.push_back(rop); - if self.undeclared_labels.is_empty() { - self.choose_sizes()?; - } - } - (_, RawOp::Op(AbstractOp::MacroDefinition(defn))) => { - if let Some(RawOp::Op(AbstractOp::Macro(invc))) = self.pending.front() { - if defn.name() == &invc.name { - let invc = invc.clone(); - self.pending.pop_front(); - self.expand_macro(&invc.name, &invc.parameters)?; - } - } - } - (_, rop) => { - // Not a label. - self.pending.push_back(rop); - } - } - - // Repeatedly check if the front of the pending list is ready. - while let Some(next) = self.pending.front() { - let op = match next { - RawOp::Op(AbstractOp::Push(Imm { - tree: Expression::Terminal(Terminal::Label(_)), - .. - })) => { - if self.undeclared_labels.is_empty() { - unreachable!() - } else { - // Still waiting on more labels. - break; - } - } - RawOp::Op(AbstractOp::Label(_)) => unreachable!(), - RawOp::Op(AbstractOp::Macro(_)) => { - // Still waiting on more macros. - break; - } - RawOp::Op(op) => op, - RawOp::Raw(_) => { - self.pop_pending()?; - continue; - } - }; - - match op - .clone() - .concretize((&self.declared_labels, &self.declared_macros).into()) - { - Ok(cop) => { - let front = self.pending.front_mut().unwrap(); - *front = RawOp::Op(cop.into()); - } - Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { - return error::ExpressionTooLarge { - expr: op.expr().unwrap().clone(), - value, - spec, - } - .fail(); - } - Err(_) => { - // Still waiting for some definition. - break; - } - } - - self.pop_pending()?; - } - - Ok(()) - } - - fn pop_pending(&mut self) -> Result<(), Error> { - let popped = self.pending.pop_front().unwrap(); - - let size; - - match popped { - RawOp::Raw(raw) => { - size = raw.len(); - self.ready.extend(raw); - } - RawOp::Op(aop) => { - let cop = aop - .concretize((&self.declared_labels, &self.declared_macros).into()) - // Already able to concretize in `push_pending` loop. - .unwrap(); - size = cop.size(); - cop.assemble(&mut self.ready); - } - } - - self.concrete_len += size; - - if self.pending.is_empty() { - self.pending_len = Some(0); - } else if let Some(ref mut pending_len) = self.pending_len { - *pending_len -= size; - } - - Ok(()) - } - - fn choose_sizes(&mut self) -> Result<(), Error> { - let mut sizes: HashMap> = self - .pending - .iter() - .filter(|op| matches!(op, RawOp::Op(AbstractOp::Push(_)))) - .map(|op| (op.expr().unwrap().clone(), Op::<()>::push_for(1).unwrap())) - .collect(); - - let mut subasm; - - loop { - // Create a sub-assembler to try assembling with the sizes in - // `undefined_labels`. - subasm = Self::default(); - subasm.concrete_len = self.concrete_len; - subasm.declared_labels = self.declared_labels.clone(); - - let result: Result, Error> = self - .pending - .iter() - .map(|op| { - let new = match op { - RawOp::Op(AbstractOp::Push(Imm { tree, .. })) => { - let new = sizes[tree].with(tree.clone()).unwrap(); - let aop = AbstractOp::new(new); - RawOp::Op(aop) - } - op => op.clone(), - }; - - subasm.push_pending(new) - }) - .collect(); - - match result { - Ok(_) => { - assert!(subasm.pending.is_empty()); - break; - } - Err(Error::ExpressionTooLarge { expr, .. }) => { - // If an expression is too large for an op, increase the width of that op. - let item = sizes.get_mut(&expr).unwrap(); - let new_size = item.upsize().context(error::UnsizedPushTooLarge)?; - *item = new_size; - } - Err(e) => return Err(e), - } - } - - // Insert the results of the sub-assembler into self. - let raw = subasm.take(); - self.pending_len = Some(raw.len()); - self.pending.clear(); - self.pending.push_back(RawOp::Raw(raw)); - self.declared_labels = subasm.declared_labels; - - Ok(()) - } - fn expand_macro( &mut self, name: &str, @@ -676,18 +521,17 @@ impl Assembler { Ok(Some(self.push_all(m.contents)?)) } _ => { - self.pending.push_back(RawOp::Op(AbstractOp::Macro( - ops::InstructionMacroInvocation { - name: name.to_string(), - parameters: parameters.to_vec(), - }, - ))); + self.undefined_macros.push(PendingMacro { + name: name.to_owned(), + offset: self.ready.len(), + }); Ok(None) } } } } +/* #[cfg(test)] mod tests { use super::*; @@ -1385,3 +1229,5 @@ mod tests { Ok(()) } } + + */ diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 36fc72cf..1a56b71f 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -276,3 +276,33 @@ fn every_op() -> Result<(), Error> { Ok(()) } + +#[test] +fn every_op2() -> Result<(), Error> { + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + ingester.ingest_file(source(&["every-op", "test.etk"]))?; + + //assert_eq!( + // output, + // hex!( + // " + // 00 + //" + // ) + //); + + // 5b: jumpdest + // 60: push1 + // 56: jump + + //label1: + //jumpdest + // push1 0x01 + + //push1 label1 + //jump + println!("{:x?}", output); + + Ok(()) +} diff --git a/etk-asm/tests/asm/every-op/test.etk b/etk-asm/tests/asm/every-op/test.etk new file mode 100644 index 00000000..df360571 --- /dev/null +++ b/etk-asm/tests/asm/every-op/test.etk @@ -0,0 +1,7 @@ +push1 label1 +jump + +label1: + jumpdest + push1 0x01 + From 27321ce6e256c83bf40ccb0bf5843aa10599147d Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Wed, 13 Sep 2023 17:33:05 -0300 Subject: [PATCH 04/48] Issue with internal labels (macros) --- etk-asm/src/asm.rs | 24 ++++++++++-------- etk-asm/src/ingest.rs | 13 ++++++++++ etk-asm/tests/asm.rs | 7 +++++- etk-asm/tests/asm/every-op/test.etk | 39 +++++++++++++++++++++++++---- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 49e93428..16c7037b 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -336,6 +336,7 @@ impl Assembler { } hash_map::Entry::Vacant(v) => { v.insert(None); + self.undefined_labels.retain(|l| l.label != *label); } } } @@ -355,20 +356,21 @@ impl Assembler { } } } - _ => { - println!("declare_content (salio): {:?}", rop) - } + _ => {} }; // Get all labels used by `rop`, check if they've been defined, and if not, note them as // "undeclared". - //if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { - // for label in labels { - // if !self.declared_labels.contains_key(&label) { - // self.undefined_labels.insert(label.to_owned()); - // } - // } - //} + if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { + for label in labels { + if !self.declared_labels.contains_key(&label) { + self.undefined_labels.push(PendingLabel { + label: label.to_owned(), + offset: self.ready.len(), + }); + } + } + } Ok(()) } @@ -381,6 +383,7 @@ impl Assembler { O: Into, { let rop = rop.into(); + println!("pushing {:?}", rop); self.declare_content(&rop)?; @@ -484,6 +487,7 @@ impl Assembler { // First pass, find locally defined labels and rename them. for op in m.contents.iter_mut() { + println!("macro {:?} op: {:?}", name, op); match op { AbstractOp::Label(ref mut label) => { let mangled = format!("{}_{}_{}", m.name, label, rng.gen::()); diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index 05227609..c1c8577a 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -314,6 +314,17 @@ where } } + fn finish(&mut self) { + for frame in self.sources.iter_mut().rev() { + let asm = match frame.scope { + Scope::Same => continue, + Scope::Independent(ref mut a) => a, + }; + + asm.finish().unwrap(); + } + } + fn write(&mut self, mut op: RawOp) -> Result<(), Error> { if self.sources.is_empty() { panic!("no sources!"); @@ -462,6 +473,8 @@ where } } + self.sources.finish(); + if !self.sources.sources.is_empty() { panic!("extra sources?"); } diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 1a56b71f..25a0fb75 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -75,6 +75,8 @@ fn instruction_macro() -> Result<(), Error> { let mut ingester = Ingest::new(&mut output); ingester.ingest_file(source(&["instruction-macro", "main.etk"]))?; + println!("{:x?}", output); + assert_eq!( output, hex!("5b5860005660406005600d601561000061004260086100006100426008") @@ -297,12 +299,15 @@ fn every_op2() -> Result<(), Error> { // 56: jump //label1: - //jumpdest + // djumpdest // push1 0x01 //push1 label1 //jump + + // [91, 88, 96, 0, 86, 96, 64, 96, 5, 96, 3|13, 96, 17|21, 97, 0, 0, 97, 0, 66, 96, 8, 97, 0, 0, 97, 0, 66, 96, 8] println!("{:x?}", output); + println!("{:?}", output); Ok(()) } diff --git a/etk-asm/tests/asm/every-op/test.etk b/etk-asm/tests/asm/every-op/test.etk index df360571..1fa0c7b0 100644 --- a/etk-asm/tests/asm/every-op/test.etk +++ b/etk-asm/tests/asm/every-op/test.etk @@ -1,7 +1,36 @@ -push1 label1 -jump +%macro foo() + push1 start + jump +%end -label1: - jumpdest - push1 0x01 +# // [91, 88, 96, 0, 86, 96, 64, 96, 5, 96, 3|13, 96, 17|21, 97, 0, 0, 97, 0, 66, 96, 8, 97, 0, 0, 97, 0, 66, 96, 8] + +%macro bar() + start: + push1 0x40 + push1 start +%end + +%macro foobar(tag, gat) + push2 $tag + push2 $gat + push1 delayed_expr(7) +%end + +%def delayed_expr(x) + $x + 1 +%end + +start: + jumpdest + pc + %foo() + %bar() + push1 delayed + push1 delayed2 + +delayed: + %foobar(start, 0x42) +delayed2: + %foobar(start, 0x42) From 95a2fd4cd5c934489230da50bca5dbeb3e13e873 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 14 Sep 2023 15:15:22 -0300 Subject: [PATCH 05/48] Some tests passing. Issue with includes --- etk-asm/src/asm.rs | 9 +++++-- etk-asm/tests/asm.rs | 9 +++++++ etk-asm/tests/asm/every-op/test.etk | 39 +++-------------------------- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 16c7037b..2449947a 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -361,7 +361,7 @@ impl Assembler { // Get all labels used by `rop`, check if they've been defined, and if not, note them as // "undeclared". - if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { + /*if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { for label in labels { if !self.declared_labels.contains_key(&label) { self.undefined_labels.push(PendingLabel { @@ -370,7 +370,7 @@ impl Assembler { }); } } - } + }*/ Ok(()) } @@ -437,6 +437,11 @@ impl Assembler { } Err(ops::Error::ContextIncomplete { source }) => match source { UnknownLabel { label, .. } => { + let size = match op.size() { + Some(size) => size, + None => 2, // push + label + }; + self.concrete_len += size; self.ready.push(rop); self.undefined_labels.push(PendingLabel { label: label.to_owned(), diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 25a0fb75..5487409d 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -308,6 +308,15 @@ fn every_op2() -> Result<(), Error> { // [91, 88, 96, 0, 86, 96, 64, 96, 5, 96, 3|13, 96, 17|21, 97, 0, 0, 97, 0, 66, 96, 8, 97, 0, 0, 97, 0, 66, 96, 8] println!("{:x?}", output); println!("{:?}", output); + println!( + "Expected {:?}", + hex!("5b5860005660406005600d601561000061004260086100006100426008") + ); + + assert_eq!( + output, + hex!("5b5860005660406005600d601561000061004260086100006100426008") + ); Ok(()) } diff --git a/etk-asm/tests/asm/every-op/test.etk b/etk-asm/tests/asm/every-op/test.etk index 1fa0c7b0..e1df3285 100644 --- a/etk-asm/tests/asm/every-op/test.etk +++ b/etk-asm/tests/asm/every-op/test.etk @@ -1,36 +1,5 @@ -%macro foo() - push1 start - jump -%end - -# // [91, 88, 96, 0, 86, 96, 64, 96, 5, 96, 3|13, 96, 17|21, 97, 0, 0, 97, 0, 66, 96, 8, 97, 0, 0, 97, 0, 66, 96, 8] - -%macro bar() - start: - push1 0x40 - push1 start -%end - -%macro foobar(tag, gat) - push2 $tag - push2 $gat - push1 delayed_expr(7) -%end - -%def delayed_expr(x) - $x + 1 -%end - -start: - jumpdest - pc - %foo() - %bar() - push1 delayed - push1 delayed2 - -delayed: - %foobar(start, 0x42) -delayed2: - %foobar(start, 0x42) +%push(hello) +jump +hello: +jumpdest \ No newline at end of file From ea5a59073c71eead4614660bb9ca065512f461a8 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 15 Sep 2023 12:09:08 -0300 Subject: [PATCH 06/48] All ASM test passing --- etk-asm/src/asm.rs | 15 +++++++++++---- etk-asm/src/ingest.rs | 8 ++++---- etk-asm/tests/asm.rs | 19 ++++++------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 2449947a..d39aae3a 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -265,7 +265,11 @@ impl Assembler { pub fn take(&mut self) -> Vec { let output = self.finish(); match output { - Ok(v) => std::mem::take(&mut v.clone()), + Ok(v) => { + self.ready.clear(); + self.concrete_len = 0; + v + } Err(e) => { eprintln!("error: {}", e); Vec::new() @@ -296,7 +300,7 @@ impl Assembler { // Concretize every RawOp ready, and collect the results. let mut output = Vec::new(); for ready_op in self.ready.iter() { - if let &RawOp::Op(ref op) = ready_op { + if let RawOp::Op(ref op) = ready_op { match op .clone() .concretize((&self.declared_labels, &self.declared_macros).into()) @@ -304,6 +308,9 @@ impl Assembler { Ok(cop) => cop.assemble(&mut output), Err(e) => unreachable!("all ops should be concretizable: {}", e), } + } else if let RawOp::Raw(raw) = ready_op { + self.concrete_len += raw.len(); + output.extend(raw); } } @@ -392,11 +399,11 @@ impl Assembler { // individually which calls the correct unchecked push. if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { self.expand_macro(&m.name, &m.parameters)?; - return Ok(self.ready.len()); + return Ok(self.concrete_len); } self.push_ready(rop)?; - Ok(self.ready.len()) + Ok(self.concrete_len) } fn push_ready(&mut self, rop: RawOp) -> Result<(), Error> { diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index c1c8577a..9bc91adc 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -297,7 +297,7 @@ where }; let raw = asm.take(); - asm.finish()?; + //asm.finish()?; if raw.is_empty() { return Ok(()); @@ -314,7 +314,7 @@ where } } - fn finish(&mut self) { + /*fn finish(&mut self) { for frame in self.sources.iter_mut().rev() { let asm = match frame.scope { Scope::Same => continue, @@ -323,7 +323,7 @@ where asm.finish().unwrap(); } - } + }*/ fn write(&mut self, mut op: RawOp) -> Result<(), Error> { if self.sources.is_empty() { @@ -473,7 +473,7 @@ where } } - self.sources.finish(); + //self.sources.finish(); if !self.sources.sources.is_empty() { panic!("extra sources?"); diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 5487409d..80346d1b 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -283,7 +283,7 @@ fn every_op() -> Result<(), Error> { fn every_op2() -> Result<(), Error> { let mut output = Vec::new(); let mut ingester = Ingest::new(&mut output); - ingester.ingest_file(source(&["every-op", "test.etk"]))?; + ingester.ingest_file(source(&["subdirectory", "main.etk"]))?; //assert_eq!( // output, @@ -305,18 +305,11 @@ fn every_op2() -> Result<(), Error> { //push1 label1 //jump - // [91, 88, 96, 0, 86, 96, 64, 96, 5, 96, 3|13, 96, 17|21, 97, 0, 0, 97, 0, 66, 96, 8, 97, 0, 0, 97, 0, 66, 96, 8] - println!("{:x?}", output); - println!("{:?}", output); - println!( - "Expected {:?}", - hex!("5b5860005660406005600d601561000061004260086100006100426008") - ); - - assert_eq!( - output, - hex!("5b5860005660406005600d601561000061004260086100006100426008") - ); + // [99, 192, 1, 192, 222] == [99, 192, 1, 192, 222, 96, 255] + println!("Output (hex) {:x?}", output); + println!("Output (dec) {:?}", output); + println!("Expected (dec) {:?}", hex!("63c001c0de60ff")); + assert_eq!(output, hex!("63c001c0de60ff")); Ok(()) } From ea1668b517099d03684d561c65afeaf0e6c21efe Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 15 Sep 2023 19:31:14 -0300 Subject: [PATCH 07/48] Fixing delayed Macros by insertion. Issue with misplaced internal labels (WIP) --- etk-asm/src/asm.rs | 90 ++++++++++++++++++++++++++++--------------- etk-asm/src/ingest.rs | 4 +- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index d39aae3a..ac1c5cd7 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -238,6 +238,9 @@ pub struct PendingMacro { /// The name of the macro. name: String, + /// Parameters of the macro. + parameters: Vec, + /// The offset of the macro. offset: usize, } @@ -326,10 +329,10 @@ impl Assembler { O: Into, { for op in ops { - self.push(op)?; + self.push(op, None)?; } - Ok(self.ready.len()) + Ok(self.concrete_len) } /// Insert explicilty declared macros and labels, via `AbstractOp`, and implictly declared @@ -354,12 +357,6 @@ impl Assembler { } hash_map::Entry::Vacant(v) => { v.insert(defn.to_owned()); - - for m in self.undefined_macros.iter() { - if m.name == *defn.name() { - self.ready.insert(m.offset, rop.clone()); - } - } } } } @@ -385,7 +382,7 @@ impl Assembler { /// Feed a single instruction into the `Assembler`. /// /// Returns the number of bytes that can be collected with [`Assembler::take`] - pub fn push(&mut self, rop: O) -> Result + pub fn push(&mut self, rop: O, pos: Option) -> Result where O: Into, { @@ -398,15 +395,15 @@ impl Assembler { // regardless if we `push_read` or `push_pending` -- in fact, `expand_macro` pushes each op // individually which calls the correct unchecked push. if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { - self.expand_macro(&m.name, &m.parameters)?; + self.expand_macro(&m.name, &m.parameters, None)?; return Ok(self.concrete_len); } - self.push_ready(rop)?; + self.push_ready(rop, pos)?; Ok(self.concrete_len) } - fn push_ready(&mut self, rop: RawOp) -> Result<(), Error> { + fn push_ready(&mut self, rop: RawOp, pos: Option) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(label)) => { let old = self @@ -416,7 +413,26 @@ impl Assembler { assert_eq!(old, None, "label should have been undefined"); Ok(()) } - RawOp::Op(AbstractOp::MacroDefinition(_)) => Ok(()), + RawOp::Op(AbstractOp::MacroDefinition(ref defn)) => { + let macros_to_expand: Vec<_> = self + .undefined_macros + .iter() + .filter(|um| um.name == *defn.name()) + .map(|um| (um.name.clone(), um.parameters.clone(), um.offset)) + .collect(); + + let expanded_names: HashSet<_> = + macros_to_expand.iter().map(|(name, _, _)| name).collect(); + + self.undefined_macros + .retain(|um| !expanded_names.contains(&um.name)); + + for (name, parameters, offset) in macros_to_expand.iter() { + self.expand_macro(name, parameters, Some(*offset))?; + } + + Ok(()) + } RawOp::Op(AbstractOp::Macro(ref m)) => Ok(()), RawOp::Op(ref op) => { match op @@ -425,7 +441,11 @@ impl Assembler { { Ok(cop) => { self.concrete_len += cop.size(); - self.ready.push(rop); + match pos { + Some(pos) => _ = self.ready.insert(pos, rop), + None => self.ready.push(rop), + } + //self.ready.push(rop); } Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { return error::ExpressionTooLarge { @@ -446,7 +466,7 @@ impl Assembler { UnknownLabel { label, .. } => { let size = match op.size() { Some(size) => size, - None => 2, // push + label + None => 2, // %push(label) }; self.concrete_len += size; self.ready.push(rop); @@ -455,14 +475,15 @@ impl Assembler { offset: self.concrete_len, }); } - UnknownMacro { name, .. } => { - self.ready.push(rop); - self.undefined_macros.push(PendingMacro { - name: name.to_owned(), - offset: self.concrete_len, - }); - } - UndefinedVariable { name, .. } => todo!(), + UnknownMacro { name, .. } => todo!("unknown macro {}", name), + //{ + // self.ready.push(rop); + // self.undefined_macros.push(PendingMacro { + // name: name.to_owned(), + // offset: self.concrete_len, + // }); + //} + UndefinedVariable { name, .. } => todo!("undefined variable {}", name), }, } @@ -470,7 +491,11 @@ impl Assembler { } RawOp::Raw(raw) => { self.concrete_len += raw.len(); - self.ready.push(RawOp::Raw(raw)); + match pos { + Some(pos) => _ = self.ready.splice(pos..pos, vec![RawOp::Raw(raw)]), + None => self.ready.push(RawOp::Raw(raw)), + } + //self.ready.push(RawOp::Raw(raw)); Ok(()) } } @@ -480,6 +505,7 @@ impl Assembler { &mut self, name: &str, parameters: &[Expression], + position: Option, ) -> Result, Error> { // Remap labels to macro scope. match self.declared_macros.get(name).cloned() { @@ -534,11 +560,18 @@ impl Assembler { } } - Ok(Some(self.push_all(m.contents)?)) + let mut pushed_size = 0; + for op in m.contents.iter().rev() { + let ps = self.push(op.clone(), position)?; + pushed_size += ps; + } + + Ok(Some(pushed_size)) } _ => { self.undefined_macros.push(PendingMacro { name: name.to_owned(), + parameters: parameters.to_owned(), offset: self.ready.len(), }); Ok(None) @@ -547,7 +580,6 @@ impl Assembler { } } -/* #[cfg(test)] mod tests { use super::*; @@ -708,9 +740,9 @@ mod tests { #[test] fn assemble_variable_push2() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push(AbstractOp::Push(Imm::with_label("auto")))?; + asm.push(AbstractOp::Push(Imm::with_label("auto")), None)?; for _ in 0..255 { - asm.push(AbstractOp::new(GetPc))?; + asm.push(AbstractOp::new(GetPc), None)?; } asm.push_all(vec![ @@ -1245,5 +1277,3 @@ mod tests { Ok(()) } } - - */ diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index 9bc91adc..b0be48b0 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -336,7 +336,7 @@ where Scope::Independent(ref mut a) => a, }; - if 0 == asm.push(op)? { + if 0 == asm.push(op, None)? { return Ok(()); } else { op = RawOp::Raw(asm.take()); @@ -348,7 +348,7 @@ where Scope::Same => panic!("sources[0] must be independent"), }; - first_asm.push(op)?; + first_asm.push(op, None)?; Ok(()) } From 53e12a34c079e5ca3810e9756f49eb3a82eb5582 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 15 Sep 2023 20:46:25 -0300 Subject: [PATCH 08/48] Progress in Macro delayed, 3/34 tests failing --- etk-asm/src/asm.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index ac1c5cd7..03448eb1 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -560,19 +560,24 @@ impl Assembler { } } - let mut pushed_size = 0; - for op in m.contents.iter().rev() { - let ps = self.push(op.clone(), position)?; - pushed_size += ps; - } + match position { + Some(pos) => { + let actual_size = self.ready.len(); + for op in m.contents.iter() { + let offset = self.ready.len() - actual_size + pos; + self.push(op.clone(), Some(offset))?; + } - Ok(Some(pushed_size)) + Ok(Some(self.concrete_len)) + } + None => Ok(Some(self.push_all(m.contents)?)), + } } _ => { self.undefined_macros.push(PendingMacro { name: name.to_owned(), parameters: parameters.to_owned(), - offset: self.ready.len(), + offset: self.concrete_len, }); Ok(None) } From 4df519980242bdd99376642556b8e1b74c31b312 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 18 Sep 2023 11:39:46 -0300 Subject: [PATCH 09/48] Tests: 39 passed; 2 failed; Need to find a way to compute the size of "push label" ops --- etk-asm/src/asm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 03448eb1..c0695e85 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -472,7 +472,7 @@ impl Assembler { self.ready.push(rop); self.undefined_labels.push(PendingLabel { label: label.to_owned(), - offset: self.concrete_len, + offset: self.ready.len(), }); } UnknownMacro { name, .. } => todo!("unknown macro {}", name), @@ -577,7 +577,7 @@ impl Assembler { self.undefined_macros.push(PendingMacro { name: name.to_owned(), parameters: parameters.to_owned(), - offset: self.concrete_len, + offset: self.ready.len(), }); Ok(None) } From 2be1ee3073daba2d0e51014c0140ef73e3019778 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 18 Sep 2023 18:21:06 -0300 Subject: [PATCH 10/48] ingest_import_in_include failing. Need to check concrete_len during include/import --- etk-asm/src/asm.rs | 32 ++++++++++++++++++++------------ etk-asm/tests/asm.rs | 35 ----------------------------------- 2 files changed, 20 insertions(+), 47 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index c0695e85..14c1de19 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -136,6 +136,7 @@ use crate::ops::{self, AbstractOp, Assemble, Expression, Imm, MacroDefinition}; use etk_ops::cancun::{Op, Push1}; use rand::Rng; use snafu::OptionExt; +use std::cmp; use std::collections::{hash_map, HashMap, HashSet, VecDeque}; /// An item to be assembled, which can be either an [`AbstractOp`] or a raw byte @@ -346,7 +347,6 @@ impl Assembler { } hash_map::Entry::Vacant(v) => { v.insert(None); - self.undefined_labels.retain(|l| l.label != *label); } } } @@ -365,7 +365,7 @@ impl Assembler { // Get all labels used by `rop`, check if they've been defined, and if not, note them as // "undeclared". - /*if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { + if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { for label in labels { if !self.declared_labels.contains_key(&label) { self.undefined_labels.push(PendingLabel { @@ -374,7 +374,7 @@ impl Assembler { }); } } - }*/ + } Ok(()) } @@ -406,9 +406,19 @@ impl Assembler { fn push_ready(&mut self, rop: RawOp, pos: Option) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(label)) => { + let mut dst = 0; + for ul in self.undefined_labels.iter() { + if ul.label == label { + let tmp = ((self.concrete_len - ul.offset) / 256) as f32; + dst = cmp::max(tmp.floor() as usize, dst) + } + } + + self.undefined_labels.retain(|l| l.label != *label); + let old = self .declared_labels - .insert(label, Some(self.concrete_len)) + .insert(label, Some(self.concrete_len + dst)) .expect("label should exist"); assert_eq!(old, None, "label should have been undefined"); Ok(()) @@ -433,7 +443,7 @@ impl Assembler { Ok(()) } - RawOp::Op(AbstractOp::Macro(ref m)) => Ok(()), + RawOp::Op(AbstractOp::Macro(_)) => Ok(()), RawOp::Op(ref op) => { match op .clone() @@ -445,7 +455,6 @@ impl Assembler { Some(pos) => _ = self.ready.insert(pos, rop), None => self.ready.push(rop), } - //self.ready.push(rop); } Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { return error::ExpressionTooLarge { @@ -466,14 +475,14 @@ impl Assembler { UnknownLabel { label, .. } => { let size = match op.size() { Some(size) => size, - None => 2, // %push(label) + None => 2, // %push(label) with min size }; self.concrete_len += size; self.ready.push(rop); - self.undefined_labels.push(PendingLabel { - label: label.to_owned(), - offset: self.ready.len(), - }); + //self.undefined_labels.push(PendingLabel { + // label: label.to_owned(), + // offset: self.ready.len(), + //}); } UnknownMacro { name, .. } => todo!("unknown macro {}", name), //{ @@ -495,7 +504,6 @@ impl Assembler { Some(pos) => _ = self.ready.splice(pos..pos, vec![RawOp::Raw(raw)]), None => self.ready.push(RawOp::Raw(raw)), } - //self.ready.push(RawOp::Raw(raw)); Ok(()) } } diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 80346d1b..622519f9 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -278,38 +278,3 @@ fn every_op() -> Result<(), Error> { Ok(()) } - -#[test] -fn every_op2() -> Result<(), Error> { - let mut output = Vec::new(); - let mut ingester = Ingest::new(&mut output); - ingester.ingest_file(source(&["subdirectory", "main.etk"]))?; - - //assert_eq!( - // output, - // hex!( - // " - // 00 - //" - // ) - //); - - // 5b: jumpdest - // 60: push1 - // 56: jump - - //label1: - // djumpdest - // push1 0x01 - - //push1 label1 - //jump - - // [99, 192, 1, 192, 222] == [99, 192, 1, 192, 222, 96, 255] - println!("Output (hex) {:x?}", output); - println!("Output (dec) {:?}", output); - println!("Expected (dec) {:?}", hex!("63c001c0de60ff")); - assert_eq!(output, hex!("63c001c0de60ff")); - - Ok(()) -} From b6b23ba65b351d3a077332958f9949b83b1be181 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 19 Sep 2023 11:24:58 -0300 Subject: [PATCH 11/48] cargo test: all tests passing :tada:. Code needs to be cleaned before merge --- etk-asm/src/asm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 14c1de19..8e41b06d 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -271,7 +271,7 @@ impl Assembler { match output { Ok(v) => { self.ready.clear(); - self.concrete_len = 0; + //self.concrete_len = 0; v } Err(e) => { @@ -410,7 +410,7 @@ impl Assembler { for ul in self.undefined_labels.iter() { if ul.label == label { let tmp = ((self.concrete_len - ul.offset) / 256) as f32; - dst = cmp::max(tmp.floor() as usize, dst) + dst = cmp::max(tmp.floor() as usize, dst); } } From 03a49909348f9c2c22d85e6780ddde5a24ca1e59 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 19 Sep 2023 13:51:02 -0300 Subject: [PATCH 12/48] Issue 124 has been fixed. A new test has been added to ensure that it remains correct over time. --- etk-asm/src/asm.rs | 64 +++++++++++++++---- etk-asm/src/ingest.rs | 2 +- etk-asm/tests/asm.rs | 23 ++++++- .../undefined-label-undefined-macro.etk | 6 ++ 4 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 8e41b06d..836c65da 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -202,9 +202,12 @@ impl From> for RawOp { /// ``` #[derive(Debug)] pub struct Assembler { - /// Assembled ops, ready to be taken. + /// Assembled ops, not yet ready to be taken. ready: Vec, + /// Assembled ops, ready to be taken. + output: Vec, + /// Number ob bytes in 'ready' that are ready to be taken. concrete_len: usize, @@ -224,7 +227,7 @@ pub struct Assembler { } /// TODO Temporal DOC -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PendingLabel { /// The name of the label. label: String, @@ -234,7 +237,7 @@ pub struct PendingLabel { } /// TODO Temporal DOC -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PendingMacro { /// The name of the macro. name: String, @@ -250,6 +253,7 @@ impl Default for Assembler { fn default() -> Self { Self { ready: Default::default(), + output: Default::default(), concrete_len: 0, declared_labels: Default::default(), declared_macros: Default::default(), @@ -267,11 +271,12 @@ impl Assembler { /// Collect any assembled instructions that are ready to be output. pub fn take(&mut self) -> Vec { - let output = self.finish(); + let output = self.concretize_ops(); match output { Ok(v) => { self.ready.clear(); //self.concrete_len = 0; + //self.output.clear(); v } Err(e) => { @@ -282,7 +287,45 @@ impl Assembler { } /// Temporal documentation. TODO - pub fn finish(&mut self) -> Result, Error> { + fn concretize_ops(&mut self) -> Result, Error> { + let mut output = Vec::new(); + for op in self.ready.iter() { + if let RawOp::Op(ref op) = op { + match op + .clone() + .concretize((&self.declared_labels, &self.declared_macros).into()) + { + Ok(cop) => cop.assemble(&mut output), + Err(ops::Error::ContextIncomplete { source }) => match source { + UnknownLabel { label: _label, .. } => { + let undefined_names: Vec<_> = self + .undefined_labels + .iter() + .map(|PendingLabel { label, .. }| label.clone()) + .collect(); + return error::UndeclaredLabels { + labels: undefined_names, + } + .fail(); + } + UnknownMacro { name, .. } => { + return error::UndeclaredInstructionMacro { name }.fail(); + } + + UndefinedVariable { name, .. } => todo!("undefined variable {}", name), + }, + Err(_) => unreachable!("all ops should be concretizable"), + } + } else if let RawOp::Raw(raw) = op { + output.extend(raw); + } + } + + Ok(output) + } + + /// Temporal documentation. TODO + pub fn finish(&mut self) -> Result<(), Error> { if !self.undefined_labels.is_empty() { return error::UndeclaredLabels { labels: self @@ -302,23 +345,22 @@ impl Assembler { } // Concretize every RawOp ready, and collect the results. - let mut output = Vec::new(); - for ready_op in self.ready.iter() { + /*for ready_op in self.ready.iter() { if let RawOp::Op(ref op) = ready_op { match op .clone() .concretize((&self.declared_labels, &self.declared_macros).into()) { - Ok(cop) => cop.assemble(&mut output), + Ok(cop) => cop.assemble(&mut self.output), Err(e) => unreachable!("all ops should be concretizable: {}", e), } } else if let RawOp::Raw(raw) = ready_op { self.concrete_len += raw.len(); - output.extend(raw); + self.output.extend(raw); } - } + }*/ - Ok(output) + Ok(()) } /// Feed instructions into the `Assembler`. diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index b0be48b0..dcf5922b 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -297,7 +297,7 @@ where }; let raw = asm.take(); - //asm.finish()?; + asm.finish()?; if raw.is_empty() { return Ok(()); diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 622519f9..c7ccc7f8 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -1,6 +1,9 @@ use assert_matches::assert_matches; -use etk_asm::ingest::{Error, Ingest}; +use etk_asm::{ + asm::Assembler, + ingest::{Error, Ingest}, +}; use hex_literal::hex; @@ -75,8 +78,6 @@ fn instruction_macro() -> Result<(), Error> { let mut ingester = Ingest::new(&mut output); ingester.ingest_file(source(&["instruction-macro", "main.etk"]))?; - println!("{:x?}", output); - assert_eq!( output, hex!("5b5860005660406005600d601561000061004260086100006100426008") @@ -110,6 +111,22 @@ fn instruction_macro_with_two_instructions_per_line() { assert_matches!(err, Error::Parse { .. }); } +#[test] +fn undefined_label_undefined_macro() { + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + let err = ingester + .ingest_file(source(&[ + "instruction-macro", + "undefined-label-undefined-macro.etk", + ])) + .unwrap_err(); + + assert_matches!(err, etk_asm::ingest::Error::Assemble { source: + etk_asm::asm::Error::UndeclaredLabels { labels, .. }, .. + } if labels == vec!["revert".to_string()]); +} + #[test] fn every_op() -> Result<(), Error> { let mut output = Vec::new(); diff --git a/etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk b/etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk new file mode 100644 index 00000000..4103fb6b --- /dev/null +++ b/etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk @@ -0,0 +1,6 @@ +%macro revert_if_neq() + push1 revert +%end + +%revert_if_neq() +%revert() \ No newline at end of file From 93c66c94fdd37b0451ad5995ce2c8251171fe7d7 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 19 Sep 2023 14:12:46 -0300 Subject: [PATCH 13/48] Code cleaning --- etk-asm/src/asm.rs | 79 ++++++++++++------------------------------- etk-asm/src/ingest.rs | 13 ------- etk-asm/tests/asm.rs | 5 +-- 3 files changed, 22 insertions(+), 75 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 836c65da..f0883ed8 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -131,13 +131,10 @@ mod error { pub use self::error::Error; use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; -use crate::ops::expression::{self, Terminal}; -use crate::ops::{self, AbstractOp, Assemble, Expression, Imm, MacroDefinition}; -use etk_ops::cancun::{Op, Push1}; +use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; use rand::Rng; -use snafu::OptionExt; use std::cmp; -use std::collections::{hash_map, HashMap, HashSet, VecDeque}; +use std::collections::{hash_map, HashMap, HashSet}; /// An item to be assembled, which can be either an [`AbstractOp`] or a raw byte /// sequence. @@ -205,9 +202,6 @@ pub struct Assembler { /// Assembled ops, not yet ready to be taken. ready: Vec, - /// Assembled ops, ready to be taken. - output: Vec, - /// Number ob bytes in 'ready' that are ready to be taken. concrete_len: usize, @@ -226,17 +220,17 @@ pub struct Assembler { undefined_macros: Vec, } -/// TODO Temporal DOC +/// Struct used to keep track of pending label invocations and their positions in code. #[derive(Debug, Clone)] pub struct PendingLabel { /// The name of the label. label: String, - /// The offset of the label. - offset: usize, + /// Concrete position where the label was invoked. + position: usize, } -/// TODO Temporal DOC +/// Struct used to keep track of pending macro invocations and thwey positions in code. #[derive(Debug, Clone)] pub struct PendingMacro { /// The name of the macro. @@ -245,15 +239,14 @@ pub struct PendingMacro { /// Parameters of the macro. parameters: Vec, - /// The offset of the macro. - offset: usize, + /// Concrete position where the macro was invoked. + position: usize, } impl Default for Assembler { fn default() -> Self { Self { ready: Default::default(), - output: Default::default(), concrete_len: 0, declared_labels: Default::default(), declared_macros: Default::default(), @@ -275,8 +268,6 @@ impl Assembler { match output { Ok(v) => { self.ready.clear(); - //self.concrete_len = 0; - //self.output.clear(); v } Err(e) => { @@ -286,7 +277,7 @@ impl Assembler { } } - /// Temporal documentation. TODO + /// Concretize all assembled instructions. fn concretize_ops(&mut self) -> Result, Error> { let mut output = Vec::new(); for op in self.ready.iter() { @@ -324,7 +315,8 @@ impl Assembler { Ok(output) } - /// Temporal documentation. TODO + /// Indicate that the input sequence is complete. Returns any errors that + /// may remain. pub fn finish(&mut self) -> Result<(), Error> { if !self.undefined_labels.is_empty() { return error::UndeclaredLabels { @@ -344,22 +336,6 @@ impl Assembler { .fail(); } - // Concretize every RawOp ready, and collect the results. - /*for ready_op in self.ready.iter() { - if let RawOp::Op(ref op) = ready_op { - match op - .clone() - .concretize((&self.declared_labels, &self.declared_macros).into()) - { - Ok(cop) => cop.assemble(&mut self.output), - Err(e) => unreachable!("all ops should be concretizable: {}", e), - } - } else if let RawOp::Raw(raw) = ready_op { - self.concrete_len += raw.len(); - self.output.extend(raw); - } - }*/ - Ok(()) } @@ -412,7 +388,7 @@ impl Assembler { if !self.declared_labels.contains_key(&label) { self.undefined_labels.push(PendingLabel { label: label.to_owned(), - offset: self.ready.len(), + position: self.ready.len(), }); } } @@ -429,13 +405,10 @@ impl Assembler { O: Into, { let rop = rop.into(); - println!("pushing {:?}", rop); self.declare_content(&rop)?; - // Expand instruction macros immediately. We do this here because it's the same process - // regardless if we `push_read` or `push_pending` -- in fact, `expand_macro` pushes each op - // individually which calls the correct unchecked push. + // Expand instruction macros immediately. if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { self.expand_macro(&m.name, &m.parameters, None)?; return Ok(self.concrete_len); @@ -451,7 +424,7 @@ impl Assembler { let mut dst = 0; for ul in self.undefined_labels.iter() { if ul.label == label { - let tmp = ((self.concrete_len - ul.offset) / 256) as f32; + let tmp = ((self.concrete_len - ul.position) / 256) as f32; dst = cmp::max(tmp.floor() as usize, dst); } } @@ -470,7 +443,7 @@ impl Assembler { .undefined_macros .iter() .filter(|um| um.name == *defn.name()) - .map(|um| (um.name.clone(), um.parameters.clone(), um.offset)) + .map(|um| (um.name.clone(), um.parameters.clone(), um.position)) .collect(); let expanded_names: HashSet<_> = @@ -514,26 +487,17 @@ impl Assembler { .fail() } Err(ops::Error::ContextIncomplete { source }) => match source { - UnknownLabel { label, .. } => { + UnknownLabel { label: _label, .. } => { let size = match op.size() { Some(size) => size, - None => 2, // %push(label) with min size + None => 2, // %push(label) with min size (to be updated) }; self.concrete_len += size; self.ready.push(rop); - //self.undefined_labels.push(PendingLabel { - // label: label.to_owned(), - // offset: self.ready.len(), - //}); } - UnknownMacro { name, .. } => todo!("unknown macro {}", name), - //{ - // self.ready.push(rop); - // self.undefined_macros.push(PendingMacro { - // name: name.to_owned(), - // offset: self.concrete_len, - // }); - //} + UnknownMacro { name, .. } => { + return error::UndeclaredInstructionMacro { name }.fail() + } UndefinedVariable { name, .. } => todo!("undefined variable {}", name), }, } @@ -575,7 +539,6 @@ impl Assembler { // First pass, find locally defined labels and rename them. for op in m.contents.iter_mut() { - println!("macro {:?} op: {:?}", name, op); match op { AbstractOp::Label(ref mut label) => { let mangled = format!("{}_{}_{}", m.name, label, rng.gen::()); @@ -627,7 +590,7 @@ impl Assembler { self.undefined_macros.push(PendingMacro { name: name.to_owned(), parameters: parameters.to_owned(), - offset: self.ready.len(), + position: self.ready.len(), }); Ok(None) } diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index dcf5922b..ac5d9cfe 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -314,17 +314,6 @@ where } } - /*fn finish(&mut self) { - for frame in self.sources.iter_mut().rev() { - let asm = match frame.scope { - Scope::Same => continue, - Scope::Independent(ref mut a) => a, - }; - - asm.finish().unwrap(); - } - }*/ - fn write(&mut self, mut op: RawOp) -> Result<(), Error> { if self.sources.is_empty() { panic!("no sources!"); @@ -473,8 +462,6 @@ where } } - //self.sources.finish(); - if !self.sources.sources.is_empty() { panic!("extra sources?"); } diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index c7ccc7f8..ed2884af 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -1,9 +1,6 @@ use assert_matches::assert_matches; -use etk_asm::{ - asm::Assembler, - ingest::{Error, Ingest}, -}; +use etk_asm::ingest::{Error, Ingest}; use hex_literal::hex; From a7c1d962c3b37359837325d30f90895f7be5d693 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 19 Sep 2023 14:45:41 -0300 Subject: [PATCH 14/48] push_ready renamed to push_rawop --- etk-asm/src/asm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index f0883ed8..2a9bdc18 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -414,11 +414,11 @@ impl Assembler { return Ok(self.concrete_len); } - self.push_ready(rop, pos)?; + self.push_rawop(rop, pos)?; Ok(self.concrete_len) } - fn push_ready(&mut self, rop: RawOp, pos: Option) -> Result<(), Error> { + fn push_rawop(&mut self, rop: RawOp, pos: Option) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(label)) => { let mut dst = 0; From 6f0634a8398070898ed72b5c829be70afe7b89c7 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 19 Sep 2023 20:02:10 -0300 Subject: [PATCH 15/48] Rustfmt --- etk-asm/src/asm.rs | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 2a9bdc18..18b2f216 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -149,13 +149,6 @@ pub enum RawOp { } impl RawOp { - fn size(&self) -> Option { - match self { - Self::Op(op) => op.size(), - Self::Raw(raw) => Some(raw.len()), - } - } - fn expr(&self) -> Option<&Expression> { match self { Self::Op(op) => op.expr(), @@ -197,7 +190,7 @@ impl From> for RawOp { /// # assert_eq!(output, hex!("58")); /// # Result::<(), Error>::Ok(()) /// ``` -#[derive(Debug)] +#[derive(Debug, Default)] pub struct Assembler { /// Assembled ops, not yet ready to be taken. ready: Vec, @@ -243,19 +236,6 @@ pub struct PendingMacro { position: usize, } -impl Default for Assembler { - fn default() -> Self { - Self { - ready: Default::default(), - concrete_len: 0, - declared_labels: Default::default(), - declared_macros: Default::default(), - undefined_labels: Default::default(), - undefined_macros: Default::default(), - } - } -} - impl Assembler { /// Create a new `Assembler`. pub fn new() -> Self { @@ -467,7 +447,7 @@ impl Assembler { Ok(cop) => { self.concrete_len += cop.size(); match pos { - Some(pos) => _ = self.ready.insert(pos, rop), + Some(pos) => self.ready.insert(pos, rop), None => self.ready.push(rop), } } @@ -488,10 +468,7 @@ impl Assembler { } Err(ops::Error::ContextIncomplete { source }) => match source { UnknownLabel { label: _label, .. } => { - let size = match op.size() { - Some(size) => size, - None => 2, // %push(label) with min size (to be updated) - }; + let size = op.size().unwrap_or(2); // %push(label) with min size (to be updated) self.concrete_len += size; self.ready.push(rop); } From da7afe05e2ef7728a60ea49b512872955b4bb679 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 19 Sep 2023 20:42:12 -0300 Subject: [PATCH 16/48] Rustfmt in dasm (not related but fixed) --- etk-dasm/src/bin/disease.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/etk-dasm/src/bin/disease.rs b/etk-dasm/src/bin/disease.rs index 7729d9e0..08afb442 100644 --- a/etk-dasm/src/bin/disease.rs +++ b/etk-dasm/src/bin/disease.rs @@ -55,10 +55,7 @@ fn run() -> Result<(), Error> { separator.push_all(disasm.ops()); - let basic_blocks = separator - .take() - .into_iter() - .chain(separator.finish().into_iter()); + let basic_blocks = separator.take().into_iter().chain(separator.finish()); for block in basic_blocks { let mut offset = block.offset; From 5530125fa711709132a5dc49575eeb825e6a05b4 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 19 Sep 2023 22:23:53 -0300 Subject: [PATCH 17/48] Rustfmt in ecfg (not related but fixed) --- etk-analyze/src/bin/ecfg.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etk-analyze/src/bin/ecfg.rs b/etk-analyze/src/bin/ecfg.rs index a8fd7a37..7f963c3b 100644 --- a/etk-analyze/src/bin/ecfg.rs +++ b/etk-analyze/src/bin/ecfg.rs @@ -58,7 +58,7 @@ fn run() -> Result<(), Error> { let blocks = separator .take() .into_iter() - .chain(separator.finish().into_iter()) + .chain(separator.finish()) .map(|x| AnnotatedBlock::annotate(&x)); let mut cfg = ControlFlowGraph::new(blocks); From 2114ffe28ebb973c6aea7301402715f95c9a1aed Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Wed, 27 Sep 2023 10:50:06 -0300 Subject: [PATCH 18/48] Output cleaning --- etk-asm/src/asm.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 18b2f216..f4c45287 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -250,10 +250,7 @@ impl Assembler { self.ready.clear(); v } - Err(e) => { - eprintln!("error: {}", e); - Vec::new() - } + Err(_) => Vec::new(), } } From f1bb8841b85eb3af4907d1551ece6172e132632a Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 2 Oct 2023 10:59:46 -0300 Subject: [PATCH 19/48] Several changes to simplify the compiler. It's still a PoC: the code is messy and there are several hacks I need to remove. Also I should write a more rusty code. --- etk-asm/src/asm.rs | 112 ++++++++++++++++++++++++------------ etk-asm/src/ast.rs | 9 ++- etk-asm/src/parse/error.rs | 35 +++++++++++ etk-asm/src/parse/macros.rs | 47 +++++++++++++-- etk-asm/src/parse/mod.rs | 14 +++-- etk-asm/tests/asm.rs | 16 +++++- 6 files changed, 177 insertions(+), 56 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index f4c45287..c2d467c4 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -130,6 +130,7 @@ mod error { } pub use self::error::Error; +use crate::ast::Node; use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; use rand::Rng; @@ -195,7 +196,7 @@ pub struct Assembler { /// Assembled ops, not yet ready to be taken. ready: Vec, - /// Number ob bytes in 'ready' that are ready to be taken. + /// Number of bytes in 'ready' that are ready to be taken. concrete_len: usize, /// Labels, in `pending`, associated with an `AbstractOp::Label`. @@ -219,11 +220,11 @@ pub struct PendingLabel { /// The name of the label. label: String, - /// Concrete position where the label was invoked. + /// Position where the label was invoked. position: usize, } -/// Struct used to keep track of pending macro invocations and thwey positions in code. +/// Struct used to keep track of pending macro invocations and their positions in code. #[derive(Debug, Clone)] pub struct PendingMacro { /// The name of the macro. @@ -232,7 +233,7 @@ pub struct PendingMacro { /// Parameters of the macro. parameters: Vec, - /// Concrete position where the macro was invoked. + /// Position where the macro was invoked. position: usize, } @@ -242,6 +243,35 @@ impl Assembler { Self::default() } + /// TODO: Add DOC + pub fn new_internal(concrete_len: usize) -> Self { + Self { + concrete_len, + ..Self::default() + } + } + + /// TODO: Add DOC + pub fn get_concrete_len(&self) -> usize { + self.concrete_len + } + + /// TODO: Add DOC + pub fn inspect_macros(&mut self, nodes: I) -> Result<(), Error> + where + I: IntoIterator, + O: Into, + { + for op in nodes { + let op = op.into(); + if let RawOp::Op(AbstractOp::MacroDefinition(_)) = op { + self.declare_content(op)? + } + } + + Ok(()) + } + /// Collect any assembled instructions that are ready to be output. pub fn take(&mut self) -> Vec { let output = self.concretize_ops(); @@ -333,7 +363,11 @@ impl Assembler { /// Insert explicilty declared macros and labels, via `AbstractOp`, and implictly declared /// macros and labels via usage in `Op`. - fn declare_content(&mut self, rop: &RawOp) -> Result<(), Error> { + pub fn declare_content(&mut self, rop: O) -> Result<(), Error> + where + O: Into, + { + let rop = rop.into(); match rop { RawOp::Op(AbstractOp::Label(ref label)) => { match self.declared_labels.entry(label.to_owned()) { @@ -382,8 +416,19 @@ impl Assembler { O: Into, { let rop = rop.into(); + println!("pushing {:?}", rop); - self.declare_content(&rop)?; + //self.declare_content(&rop)?; + if let RawOp::Op(AbstractOp::Label(ref label)) = rop { + match self.declared_labels.entry(label.to_owned()) { + hash_map::Entry::Occupied(_) => { + return error::DuplicateLabel { label }.fail(); + } + hash_map::Entry::Vacant(v) => { + v.insert(None); + } + } + } // Expand instruction macros immediately. if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { @@ -415,26 +460,7 @@ impl Assembler { assert_eq!(old, None, "label should have been undefined"); Ok(()) } - RawOp::Op(AbstractOp::MacroDefinition(ref defn)) => { - let macros_to_expand: Vec<_> = self - .undefined_macros - .iter() - .filter(|um| um.name == *defn.name()) - .map(|um| (um.name.clone(), um.parameters.clone(), um.position)) - .collect(); - - let expanded_names: HashSet<_> = - macros_to_expand.iter().map(|(name, _, _)| name).collect(); - - self.undefined_macros - .retain(|um| !expanded_names.contains(&um.name)); - - for (name, parameters, offset) in macros_to_expand.iter() { - self.expand_macro(name, parameters, Some(*offset))?; - } - - Ok(()) - } + RawOp::Op(AbstractOp::MacroDefinition(_)) => Ok(()), RawOp::Op(AbstractOp::Macro(_)) => Ok(()), RawOp::Op(ref op) => { match op @@ -467,6 +493,11 @@ impl Assembler { UnknownLabel { label: _label, .. } => { let size = op.size().unwrap_or(2); // %push(label) with min size (to be updated) self.concrete_len += size; + + self.undefined_labels.push(PendingLabel { + label: _label.to_owned(), + position: self.ready.len(), + }); self.ready.push(rop); } UnknownMacro { name, .. } => { @@ -555,19 +586,12 @@ impl Assembler { self.push(op.clone(), Some(offset))?; } - Ok(Some(self.concrete_len)) + Ok(Some(self.ready.len())) } None => Ok(Some(self.push_all(m.contents)?)), } } - _ => { - self.undefined_macros.push(PendingMacro { - name: name.to_owned(), - parameters: parameters.to_owned(), - position: self.ready.len(), - }); - Ok(None) - } + _ => return error::UndeclaredInstructionMacro { name }.fail(), } } } @@ -897,6 +921,7 @@ mod tests { ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(sz, 0); let out = asm.take(); @@ -933,6 +958,7 @@ mod tests { ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(sz, 13); let out = asm.take(); @@ -965,6 +991,7 @@ mod tests { ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(sz, 8); let out = asm.take(); @@ -997,6 +1024,7 @@ mod tests { ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(sz, 8); let out = asm.take(); @@ -1029,6 +1057,7 @@ mod tests { ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(7, sz); assert_eq!(asm.take(), hex!("5b600560065858")); @@ -1042,8 +1071,10 @@ mod tests { InstructionMacroInvocation::with_zero_parameters("my_macro".into()), )]; let mut asm = Assembler::new(); - asm.push_all(ops)?; - let err = asm.finish().unwrap_err(); + asm.inspect_macros(ops.clone())?; + let err = asm.push_all(ops).unwrap_err(); + //let err = asm.finish().unwrap_err(); + println!("{:?}", err); assert_matches!(err, Error::UndeclaredInstructionMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1066,7 +1097,8 @@ mod tests { .into(), ]; let mut asm = Assembler::new(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.inspect_macros(ops.clone()).unwrap_err(); + //let err = asm.push_all(ops).unwrap_err(); assert_matches!(err, Error::DuplicateMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1086,6 +1118,7 @@ mod tests { )), ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let err = asm.push_all(ops).unwrap_err(); assert_matches!(err, Error::DuplicateLabel { label, .. } if label == "a"); @@ -1113,6 +1146,7 @@ mod tests { AbstractOp::new(Push1(Imm::with_label("a"))), ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(sz, 5); @@ -1149,6 +1183,7 @@ mod tests { ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(sz, 7); let out = asm.take(); @@ -1261,6 +1296,7 @@ mod tests { ]; let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone())?; let sz = asm.push_all(ops)?; assert_eq!(sz, 2); let out = asm.take(); diff --git a/etk-asm/src/ast.rs b/etk-asm/src/ast.rs index a4d2e15d..8c93104f 100644 --- a/etk-asm/src/ast.rs +++ b/etk-asm/src/ast.rs @@ -1,14 +1,13 @@ use crate::ops::{Abstract, AbstractOp, ExpressionMacroDefinition, InstructionMacroDefinition}; use etk_ops::cancun::Op; -use std::path::PathBuf; #[derive(Debug, Clone, PartialEq)] pub(crate) enum Node { Op(AbstractOp), - Raw(Vec), - Import(PathBuf), - Include(PathBuf), - IncludeHex(PathBuf), + //Raw(Vec), + //Import(Vec), + Include(Vec), + IncludeHex(Vec), } impl From> for Node { diff --git a/etk-asm/src/parse/error.rs b/etk-asm/src/parse/error.rs index ccc89e9e..b4e5bebb 100644 --- a/etk-asm/src/parse/error.rs +++ b/etk-asm/src/parse/error.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use pest::error::Error; use snafu::{Backtrace, IntoError, Snafu}; @@ -60,6 +62,39 @@ pub enum ParseError { /// The location of the error. backtrace: Backtrace, }, + + /// An argument provided to a macro was of the wrong type. + #[snafu(display("File {} does not exist", path.to_string_lossy()))] + #[non_exhaustive] + FileNotFound { + /// Path to the offending file. + path: PathBuf, + + /// The location of the error. + backtrace: Backtrace, + }, + + /// An included fail failed to parse as hexadecimal. + #[snafu(display("included file `{}` is invalid hex: {}", path.to_string_lossy(), source))] + #[non_exhaustive] + InvalidHex { + /// Path to the offending file. + path: PathBuf, + + /// The underlying source of this error. + source: Box, + + /// The location of the error. + backtrace: Backtrace, + }, + + /// A recursion limit was reached while including or importing a file. + #[snafu(display("too many levels of recursion/includes"))] + #[non_exhaustive] + RecursionLimit { + /// The location of the error. + backtrace: Backtrace, + }, } impl From> for ParseError { diff --git a/etk-asm/src/parse/macros.rs b/etk-asm/src/parse/macros.rs index c262917f..63633834 100644 --- a/etk-asm/src/parse/macros.rs +++ b/etk-asm/src/parse/macros.rs @@ -3,11 +3,14 @@ use super::error::ParseError; use super::expression; use super::parser::Rule; use crate::ast::Node; +use crate::ingest::Root; use crate::ops::{ AbstractOp, Expression, ExpressionMacroDefinition, ExpressionMacroInvocation, InstructionMacroDefinition, InstructionMacroInvocation, }; +use crate::parse::{error, parse_asm}; use pest::iterators::Pair; +use snafu::{ensure, IntoError}; use std::path::PathBuf; pub(crate) fn parse(pair: Pair) -> Result { @@ -22,28 +25,62 @@ pub(crate) fn parse(pair: Pair) -> Result { } } -pub(crate) fn parse_builtin(pair: Pair) -> Result { +pub(crate) fn parse_builtin( + root: Root, + pair: Pair, + mut depth: u16, +) -> Result, ParseError> { + ensure!(depth <= 255, error::RecursionLimit); let mut pairs = pair.into_inner(); let pair = pairs.next().unwrap(); assert!(pairs.next().is_none()); let rule = pair.as_rule(); + depth += 1; + let node = match rule { Rule::import => { let args = <(PathBuf,)>::parse_arguments(pair.into_inner())?; - Node::Import(args.0) + let new_path = root.canonicalized.join(args.0.clone()); + let root = Root::new(new_path.clone()).unwrap(); + let path = new_path.into_os_string().into_string().unwrap(); + let code_str = &std::fs::read_to_string(&path); + let nodes = match code_str { + Ok(code) => parse_asm(root.clone(), code, depth)?, + Err(_) => return error::FileNotFound { path: args.0 }.fail(), + }; + nodes } Rule::include => { let args = <(PathBuf,)>::parse_arguments(pair.into_inner())?; - Node::Include(args.0) + let new_path = root.canonicalized.join(args.0.clone()); + let root = Root::new(new_path.clone()).unwrap(); + let path = new_path.into_os_string().into_string().unwrap(); + let code_str = &std::fs::read_to_string(&path); + let nodes = match code_str { + Ok(code) => parse_asm(root.clone(), code, depth)?, + Err(_) => return error::FileNotFound { path: args.0 }.fail(), + }; + vec![Node::Include(nodes)] } Rule::include_hex => { let args = <(PathBuf,)>::parse_arguments(pair.into_inner())?; - Node::IncludeHex(args.0) + let file = &std::fs::read_to_string(&args.0); + + let raw = match file { + Ok(value) => hex::decode(value.trim()), + Err(_) => return error::FileNotFound { path: args.0 }.fail(), + }; + + match raw { + Ok(values) => vec![Node::IncludeHex(values)], + Err(e) => return Err(error::InvalidHex { path: args.0 }.into_error(Box::new(e))), + } + // TODO Check this error handling } Rule::push_macro => { let expr = expression::parse(pair.into_inner().next().unwrap())?; - Node::Op(AbstractOp::Push(expr.into())) + vec![Node::Op(AbstractOp::Push(expr.into()))] } _ => unreachable!(), }; diff --git a/etk-asm/src/parse/mod.rs b/etk-asm/src/parse/mod.rs index dba643c7..3d3ec02e 100644 --- a/etk-asm/src/parse/mod.rs +++ b/etk-asm/src/parse/mod.rs @@ -19,23 +19,24 @@ use self::{ error::ParseError, parser::{AsmParser, Rule}, }; -use crate::ast::Node; + use crate::ops::AbstractOp; +use crate::{ast::Node, ingest::Root}; use etk_ops::cancun::Op; use num_bigint::BigInt; use pest::{iterators::Pair, Parser}; -pub(crate) fn parse_asm(asm: &str) -> Result, ParseError> { +pub(crate) fn parse_asm(root: Root, asm: &str, depth: u16) -> Result, ParseError> { let mut program: Vec = Vec::new(); let pairs = AsmParser::parse(Rule::program, asm)?; for pair in pairs { let node = match pair.as_rule() { - Rule::builtin => macros::parse_builtin(pair)?, + Rule::builtin => macros::parse_builtin(root.clone(), pair, depth)?, Rule::EOI => continue, - _ => parse_abstract_op(pair)?.into(), + _ => vec![parse_abstract_op(pair)?.into()], }; - program.push(node); + program.extend(node); } Ok(program) @@ -77,7 +78,7 @@ fn parse_push(pair: Pair) -> Result { Ok(AbstractOp::Op(spec.with(expr).unwrap())) } - +/* #[cfg(test)] mod tests { use super::*; @@ -597,3 +598,4 @@ mod tests { assert_eq!(parse_asm(&asm).unwrap(), expected); } } + */ diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index ed2884af..3b2d5463 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -120,8 +120,8 @@ fn undefined_label_undefined_macro() { .unwrap_err(); assert_matches!(err, etk_asm::ingest::Error::Assemble { source: - etk_asm::asm::Error::UndeclaredLabels { labels, .. }, .. - } if labels == vec!["revert".to_string()]); + etk_asm::asm::Error::UndeclaredInstructionMacro { name, .. }, .. + } if name == "revert".to_string()); } #[test] @@ -292,3 +292,15 @@ fn every_op() -> Result<(), Error> { Ok(()) } + +#[test] +fn test_erc20() -> Result<(), Error> { + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + ingester.ingest_file(source(&["erc20", "Contract.etk"]))?; + + let str_out: Vec<_> = output.iter().map(|&byte| format!("{:02x}", byte)).collect(); + //assert_eq!(output, hex!("6000600060006000600060006000")); + println!("output: {:?}", str_out.concat()); + Ok(()) +} From b72d6088157b63e84b4bc9e1ba0fa37fc9d00f7f Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 2 Oct 2023 17:27:07 -0300 Subject: [PATCH 20/48] Rewriting and Simplification. Some tests still need to be solved and the code should be optimized, but the general idea seems to work. --- etk-asm/src/asm.rs | 94 ++++++------ etk-asm/src/ast.rs | 11 +- etk-asm/src/ingest.rs | 286 +++++++++++++----------------------- etk-asm/src/parse/error.rs | 8 - etk-asm/src/parse/macros.rs | 47 +----- etk-asm/src/parse/mod.rs | 13 +- 6 files changed, 161 insertions(+), 298 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index c2d467c4..2b822289 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -137,13 +137,16 @@ use rand::Rng; use std::cmp; use std::collections::{hash_map, HashMap, HashSet}; -/// An item to be assembled, which can be either an [`AbstractOp`] or a raw byte -/// sequence. +/// An item to be assembled, which can be either an [`AbstractOp`], +/// the inclusion of a new scope or a raw byte sequence. #[derive(Debug, Clone)] pub enum RawOp { /// An instruction to be assembled. Op(AbstractOp), + /// A new scope to be included. + Scope(Vec), + /// Raw bytes, for example from `%include_hex`, to be included verbatim in /// the output. Raw(Vec), @@ -153,6 +156,7 @@ impl RawOp { fn expr(&self) -> Option<&Expression> { match self { Self::Op(op) => op.expr(), + Self::Scope(_) => None, Self::Raw(_) => None, } } @@ -265,7 +269,7 @@ impl Assembler { for op in nodes { let op = op.into(); if let RawOp::Op(AbstractOp::MacroDefinition(_)) = op { - self.declare_content(op)? + self.declare_macro(op)? } } @@ -336,12 +340,12 @@ impl Assembler { .fail(); } - if !self.undefined_macros.is_empty() { + /*if !self.undefined_macros.is_empty() { return error::UndeclaredInstructionMacro { name: self.undefined_macros[0].name.to_owned(), } .fail(); - } + }*/ Ok(()) } @@ -355,7 +359,7 @@ impl Assembler { O: Into, { for op in ops { - self.push(op, None)?; + self.push(op)?; } Ok(self.concrete_len) @@ -363,22 +367,12 @@ impl Assembler { /// Insert explicilty declared macros and labels, via `AbstractOp`, and implictly declared /// macros and labels via usage in `Op`. - pub fn declare_content(&mut self, rop: O) -> Result<(), Error> + fn declare_macro(&mut self, rop: O) -> Result<(), Error> where O: Into, { let rop = rop.into(); match rop { - RawOp::Op(AbstractOp::Label(ref label)) => { - match self.declared_labels.entry(label.to_owned()) { - hash_map::Entry::Occupied(_) => { - return error::DuplicateLabel { label }.fail(); - } - hash_map::Entry::Vacant(v) => { - v.insert(None); - } - } - } RawOp::Op(AbstractOp::MacroDefinition(ref defn)) => { match self.declared_macros.entry(defn.name().to_owned()) { hash_map::Entry::Occupied(_) => { @@ -411,15 +405,27 @@ impl Assembler { /// Feed a single instruction into the `Assembler`. /// /// Returns the number of bytes that can be collected with [`Assembler::take`] - pub fn push(&mut self, rop: O, pos: Option) -> Result + pub fn push(&mut self, rop: O) -> Result where O: Into, { let rop = rop.into(); println!("pushing {:?}", rop); - //self.declare_content(&rop)?; - if let RawOp::Op(AbstractOp::Label(ref label)) = rop { + self.declare_label(&rop)?; + + // Expand instruction macros immediately. + if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { + self.expand_macro(&m.name, &m.parameters)?; + return Ok(self.concrete_len); + } + + self.push_rawop(rop)?; + Ok(self.concrete_len) + } + + fn declare_label(&mut self, rop: &RawOp) -> Result<(), Error> { + if let RawOp::Op(AbstractOp::Label(ref label)) = *rop { match self.declared_labels.entry(label.to_owned()) { hash_map::Entry::Occupied(_) => { return error::DuplicateLabel { label }.fail(); @@ -429,18 +435,10 @@ impl Assembler { } } } - - // Expand instruction macros immediately. - if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { - self.expand_macro(&m.name, &m.parameters, None)?; - return Ok(self.concrete_len); - } - - self.push_rawop(rop, pos)?; - Ok(self.concrete_len) + Ok(()) } - fn push_rawop(&mut self, rop: RawOp, pos: Option) -> Result<(), Error> { + fn push_rawop(&mut self, rop: RawOp) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(label)) => { let mut dst = 0; @@ -469,10 +467,7 @@ impl Assembler { { Ok(cop) => { self.concrete_len += cop.size(); - match pos { - Some(pos) => self.ready.insert(pos, rop), - None => self.ready.push(rop), - } + self.ready.push(rop) } Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { return error::ExpressionTooLarge { @@ -511,10 +506,15 @@ impl Assembler { } RawOp::Raw(raw) => { self.concrete_len += raw.len(); - match pos { - Some(pos) => _ = self.ready.splice(pos..pos, vec![RawOp::Raw(raw)]), - None => self.ready.push(RawOp::Raw(raw)), - } + self.ready.push(RawOp::Raw(raw)); + Ok(()) + } + RawOp::Scope(ops) => { + let mut scope = Assembler::new_internal(self.concrete_len); + scope.inspect_macros(ops.iter().cloned())?; + let sz = scope.push_all(ops)?; + self.concrete_len += sz; + self.ready.push(RawOp::Scope(scope.ready)); Ok(()) } } @@ -524,7 +524,6 @@ impl Assembler { &mut self, name: &str, parameters: &[Expression], - position: Option, ) -> Result, Error> { // Remap labels to macro scope. match self.declared_macros.get(name).cloned() { @@ -578,18 +577,7 @@ impl Assembler { } } - match position { - Some(pos) => { - let actual_size = self.ready.len(); - for op in m.contents.iter() { - let offset = self.ready.len() - actual_size + pos; - self.push(op.clone(), Some(offset))?; - } - - Ok(Some(self.ready.len())) - } - None => Ok(Some(self.push_all(m.contents)?)), - } + Ok(Some(self.push_all(m.contents)?)) } _ => return error::UndeclaredInstructionMacro { name }.fail(), } @@ -756,9 +744,9 @@ mod tests { #[test] fn assemble_variable_push2() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push(AbstractOp::Push(Imm::with_label("auto")), None)?; + asm.push(AbstractOp::Push(Imm::with_label("auto")))?; for _ in 0..255 { - asm.push(AbstractOp::new(GetPc), None)?; + asm.push(AbstractOp::new(GetPc))?; } asm.push_all(vec![ diff --git a/etk-asm/src/ast.rs b/etk-asm/src/ast.rs index 8c93104f..e32c93c0 100644 --- a/etk-asm/src/ast.rs +++ b/etk-asm/src/ast.rs @@ -1,15 +1,16 @@ +use std::path::PathBuf; + use crate::ops::{Abstract, AbstractOp, ExpressionMacroDefinition, InstructionMacroDefinition}; use etk_ops::cancun::Op; #[derive(Debug, Clone, PartialEq)] pub(crate) enum Node { Op(AbstractOp), - //Raw(Vec), - //Import(Vec), - Include(Vec), - IncludeHex(Vec), + Raw(Vec), + Import(PathBuf), + Include(PathBuf), + IncludeHex(PathBuf), } - impl From> for Node { fn from(op: Op) -> Self { Node::Op(AbstractOp::Op(op)) diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index ac5d9cfe..d98a41b2 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -106,46 +106,14 @@ use std::fs::{read_to_string, File}; use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; -fn parse_file>(path: P) -> Result, Error> { - let asm = read_to_string(path.as_ref()).with_context(|_| error::Io { - message: "reading file before parsing", - path: path.as_ref().to_owned(), - })?; - let nodes = parse_asm(&asm)?; - - Ok(nodes) -} - -#[derive(Debug)] -enum Scope { - Same, - Independent(Box), -} - -impl Scope { - fn same() -> Self { - Self::Same - } - - fn independent() -> Self { - Self::Independent(Box::new(Assembler::new())) - } -} - -#[derive(Debug)] -struct Source { - path: PathBuf, - nodes: std::vec::IntoIter, - scope: Scope, -} - -#[derive(Debug)] +#[derive(Debug, Clone)] struct Root { original: PathBuf, canonicalized: PathBuf, } impl Root { + /// TODO: Temporal DOC fn new(mut file: PathBuf) -> Result { // Pop the filename. if !file.pop() { @@ -211,135 +179,41 @@ impl Root { } } -#[must_use] -struct PartialSource<'a, W> { - stack: &'a mut SourceStack, - path: PathBuf, - scope: Scope, -} - -impl<'a, W> PartialSource<'a, W> { - fn path(&self) -> &Path { - &self.path - } - - fn push(self, nodes: Vec) -> &'a mut Source { - self.stack.sources.push(Source { - path: self.path, - nodes: nodes.into_iter(), - scope: self.scope, - }); - - self.stack.sources.last_mut().unwrap() - } -} - #[derive(Debug)] -struct SourceStack { - output: W, - sources: Vec, +struct Program { + depth: usize, root: Option, + actual_path: PathBuf, } -impl SourceStack { - fn new(output: W) -> Self { +impl Program { + fn new(root: Option) -> Self { Self { - output, - sources: Default::default(), - root: Default::default(), + depth: 0, + root: root.clone(), + actual_path: root.map(|r| r.original).unwrap_or(PathBuf::new()), } } - fn resolve(&mut self, path: PathBuf, scope: Scope) -> Result, Error> { - ensure!(self.sources.len() <= 255, error::RecursionLimit); + fn push_path(&mut self, path: PathBuf) -> Result { + ensure!(self.depth <= 255, error::RecursionLimit); + self.depth += 1; let path = if let Some(ref root) = self.root { - let last = self.sources.last().unwrap(); - let dir = match last.path.parent() { - Some(s) => s, - None => Path::new("./"), - }; - let candidate = dir.join(path); + let candidate = self.actual_path.join(path); root.check(&candidate)?; candidate } else { - assert!(self.sources.is_empty()); self.root = Some(Root::new(path.clone())?); path }; - Ok(PartialSource { - stack: self, - path, - scope, - }) + Ok(path) } - fn peek(&mut self) -> Option<&mut Source> { - self.sources.last_mut() - } -} - -impl SourceStack -where - W: Write, -{ - fn pop(&mut self) -> Result<(), Error> { - let popped = self.sources.pop().unwrap(); - - if self.sources.is_empty() { - self.root = None; - } - - let mut asm = match popped.scope { - Scope::Independent(a) => a, - Scope::Same => return Ok(()), - }; - - let raw = asm.take(); - asm.finish()?; - - if raw.is_empty() { - return Ok(()); - } - - if self.sources.is_empty() { - self.output.write_all(&raw).context(error::Io { - message: "writing output", - path: None, - })?; - Ok(()) - } else { - self.write(RawOp::Raw(raw)) - } - } - - fn write(&mut self, mut op: RawOp) -> Result<(), Error> { - if self.sources.is_empty() { - panic!("no sources!"); - } - - for frame in self.sources[1..].iter_mut().rev() { - let asm = match frame.scope { - Scope::Same => continue, - Scope::Independent(ref mut a) => a, - }; - - if 0 == asm.push(op, None)? { - return Ok(()); - } else { - op = RawOp::Raw(asm.take()); - } - } - - let first_asm = match self.sources[0].scope { - Scope::Independent(ref mut a) => a, - Scope::Same => panic!("sources[0] must be independent"), - }; - - first_asm.push(op, None)?; - - Ok(()) + fn pop_path(&mut self, oldpath: PathBuf) { + self.depth -= 1; + self.actual_path = oldpath; } } @@ -370,15 +244,13 @@ where /// ``` #[derive(Debug)] pub struct Ingest { - sources: SourceStack, + output: W, } impl Ingest { /// Make a new `Ingest` that writes assembled bytes to `output`. pub fn new(output: W) -> Self { - Self { - sources: SourceStack::new(output), - } + Self { output } } } @@ -403,69 +275,115 @@ where path: path.clone(), })?; - self.ingest(path, &text) + self.ingest(path, &text)?; + Ok(()) } - /// Assemble instructions from `src` as if they were read from a file located - /// at `path`. - pub fn ingest

(&mut self, path: P, src: &str) -> Result<(), Error> + /// TODO: Documentation + pub fn ingest

(&mut self, path: P, text: &str) -> Result<(), Error> where P: Into, { - let nodes = parse_asm(src)?; - let partial = self.sources.resolve(path.into(), Scope::independent())?; - partial.push(nodes); - - while let Some(source) = self.sources.peek() { - let node = match source.nodes.next() { - Some(n) => n, - None => { - self.sources.pop()?; - continue; - } - }; + let mut program = Program::new(Some(Root::new(path.into())?)); + let nodes = self.preprocess(&mut program, &text)?; + let mut asm = Assembler::new(); + self.run(nodes, &mut asm)?; + let raw = asm.take(); + + self.output.write_all(&raw).context(error::Io { + message: "writing output", + path: None, + })?; + + Ok(()) + } + + /// Assemble instructions from `src` as if they were read from a file located + /// at `path`. + fn preprocess(&mut self, program: &mut Program, src: &str) -> Result, Error> { + let nodes = parse_asm(&src)?; + let mut raws = Vec::new(); + for node in nodes { match node { Node::Op(op) => { - self.sources.write(RawOp::Op(op))?; + raws.push(RawOp::Op(op)); } Node::Raw(raw) => { - self.sources.write(RawOp::Raw(raw))?; + raws.push(RawOp::Raw(raw)); } - Node::Import(path) => { - let partial = self.sources.resolve(path, Scope::same())?; - let parsed = parse_file(partial.path())?; - partial.push(parsed); + Node::Import(imp_path) => { + let new_raws = self.resolve_and_ingest(program, imp_path)?; + raws.extend(new_raws); } - Node::Include(path) => { - let partial = self.sources.resolve(path, Scope::independent())?; - let parsed = parse_file(partial.path())?; - partial.push(parsed); + Node::Include(inc_path) => { + let inc_raws = self.resolve_and_ingest(program, inc_path)?; + raws.push(RawOp::Scope(inc_raws)); } - Node::IncludeHex(path) => { - let partial = self.sources.resolve(path, Scope::same())?; - + Node::IncludeHex(hex_path) => { let file = - std::fs::read_to_string(partial.path()).with_context(|_| error::Io { - message: "reading hex include", - path: partial.path().to_owned(), + std::fs::read_to_string::<&PathBuf>(&hex_path).with_context(|_| { + error::Io { + message: "reading hex include", + path: >::into(hex_path.clone()).to_owned(), + } })?; let raw = hex::decode(file.trim()) .map_err(|e| Box::new(e) as Box) .context(error::InvalidHex { - path: partial.path().to_owned(), + path: >::into(hex_path).to_owned(), })?; - partial.push(vec![Node::Raw(raw)]); + raws.push(RawOp::Raw(raw)) } } } - if !self.sources.sources.is_empty() { - panic!("extra sources?"); + Ok(raws) + } + + fn resolve_and_ingest>( + &mut self, + program: &mut Program, + path: P, + ) -> Result, Error> + where + P: Into, + { + let path = path.into(); + let oldpath = program.push_path(path.clone())?; + let code = read_to_string::<&PathBuf>(&path).with_context(|_| error::Io { + message: "reading file before parsing", + path: path.to_owned(), + })?; + let new_raws = self.preprocess(program, &code)?; + program.pop_path(oldpath); + Ok(new_raws) + } + + fn run(&mut self, ops: Vec, asm: &mut Assembler) -> Result<(), Error> { + asm.inspect_macros(ops.clone())?; + + for rawop in ops { + match rawop { + RawOp::Op(op) => { + asm.push(RawOp::Op(op))?; + } + RawOp::Scope(scope_ops) => { + let mut new_asm = Assembler::new_internal(asm.get_concrete_len()); + self.run(scope_ops, &mut new_asm)?; + let raw = new_asm.take(); + asm.push(RawOp::Raw(raw))?; + } + RawOp::Raw(hex) => { + asm.push(RawOp::Raw(hex))?; + } + } } + asm.finish()?; + Ok(()) } } diff --git a/etk-asm/src/parse/error.rs b/etk-asm/src/parse/error.rs index b4e5bebb..a60dcf13 100644 --- a/etk-asm/src/parse/error.rs +++ b/etk-asm/src/parse/error.rs @@ -87,14 +87,6 @@ pub enum ParseError { /// The location of the error. backtrace: Backtrace, }, - - /// A recursion limit was reached while including or importing a file. - #[snafu(display("too many levels of recursion/includes"))] - #[non_exhaustive] - RecursionLimit { - /// The location of the error. - backtrace: Backtrace, - }, } impl From> for ParseError { diff --git a/etk-asm/src/parse/macros.rs b/etk-asm/src/parse/macros.rs index 63633834..d3558f9b 100644 --- a/etk-asm/src/parse/macros.rs +++ b/etk-asm/src/parse/macros.rs @@ -3,14 +3,13 @@ use super::error::ParseError; use super::expression; use super::parser::Rule; use crate::ast::Node; -use crate::ingest::Root; use crate::ops::{ AbstractOp, Expression, ExpressionMacroDefinition, ExpressionMacroInvocation, InstructionMacroDefinition, InstructionMacroInvocation, }; use crate::parse::{error, parse_asm}; use pest::iterators::Pair; -use snafu::{ensure, IntoError}; +use snafu::IntoError; use std::path::PathBuf; pub(crate) fn parse(pair: Pair) -> Result { @@ -25,62 +24,28 @@ pub(crate) fn parse(pair: Pair) -> Result { } } -pub(crate) fn parse_builtin( - root: Root, - pair: Pair, - mut depth: u16, -) -> Result, ParseError> { - ensure!(depth <= 255, error::RecursionLimit); +pub(crate) fn parse_builtin(pair: Pair) -> Result { let mut pairs = pair.into_inner(); let pair = pairs.next().unwrap(); assert!(pairs.next().is_none()); let rule = pair.as_rule(); - depth += 1; - let node = match rule { Rule::import => { let args = <(PathBuf,)>::parse_arguments(pair.into_inner())?; - let new_path = root.canonicalized.join(args.0.clone()); - let root = Root::new(new_path.clone()).unwrap(); - let path = new_path.into_os_string().into_string().unwrap(); - let code_str = &std::fs::read_to_string(&path); - let nodes = match code_str { - Ok(code) => parse_asm(root.clone(), code, depth)?, - Err(_) => return error::FileNotFound { path: args.0 }.fail(), - }; - nodes + Node::Import(args.0) } Rule::include => { let args = <(PathBuf,)>::parse_arguments(pair.into_inner())?; - let new_path = root.canonicalized.join(args.0.clone()); - let root = Root::new(new_path.clone()).unwrap(); - let path = new_path.into_os_string().into_string().unwrap(); - let code_str = &std::fs::read_to_string(&path); - let nodes = match code_str { - Ok(code) => parse_asm(root.clone(), code, depth)?, - Err(_) => return error::FileNotFound { path: args.0 }.fail(), - }; - vec![Node::Include(nodes)] + Node::Include(args.0) } Rule::include_hex => { let args = <(PathBuf,)>::parse_arguments(pair.into_inner())?; - let file = &std::fs::read_to_string(&args.0); - - let raw = match file { - Ok(value) => hex::decode(value.trim()), - Err(_) => return error::FileNotFound { path: args.0 }.fail(), - }; - - match raw { - Ok(values) => vec![Node::IncludeHex(values)], - Err(e) => return Err(error::InvalidHex { path: args.0 }.into_error(Box::new(e))), - } - // TODO Check this error handling + Node::IncludeHex(args.0) } Rule::push_macro => { let expr = expression::parse(pair.into_inner().next().unwrap())?; - vec![Node::Op(AbstractOp::Push(expr.into()))] + Node::Op(AbstractOp::Push(expr.into())) } _ => unreachable!(), }; diff --git a/etk-asm/src/parse/mod.rs b/etk-asm/src/parse/mod.rs index 3d3ec02e..4858abaa 100644 --- a/etk-asm/src/parse/mod.rs +++ b/etk-asm/src/parse/mod.rs @@ -20,23 +20,23 @@ use self::{ parser::{AsmParser, Rule}, }; +use crate::ast::Node; use crate::ops::AbstractOp; -use crate::{ast::Node, ingest::Root}; use etk_ops::cancun::Op; use num_bigint::BigInt; use pest::{iterators::Pair, Parser}; -pub(crate) fn parse_asm(root: Root, asm: &str, depth: u16) -> Result, ParseError> { +pub(crate) fn parse_asm(asm: &str) -> Result, ParseError> { let mut program: Vec = Vec::new(); let pairs = AsmParser::parse(Rule::program, asm)?; for pair in pairs { let node = match pair.as_rule() { - Rule::builtin => macros::parse_builtin(root.clone(), pair, depth)?, + Rule::builtin => macros::parse_builtin(pair)?, Rule::EOI => continue, - _ => vec![parse_abstract_op(pair)?.into()], + _ => parse_abstract_op(pair)?.into(), }; - program.extend(node); + program.push(node); } Ok(program) @@ -78,7 +78,7 @@ fn parse_push(pair: Pair) -> Result { Ok(AbstractOp::Op(spec.with(expr).unwrap())) } -/* + #[cfg(test)] mod tests { use super::*; @@ -598,4 +598,3 @@ mod tests { assert_eq!(parse_asm(&asm).unwrap(), expected); } } - */ From 6319fc270517793c124aecbd8e03b92074ea6154 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 3 Oct 2023 17:27:45 -0300 Subject: [PATCH 21/48] Code reorganization. Still work in progress. --- etk-asm/src/asm.rs | 76 +-- etk-asm/src/ingest.rs | 43 +- etk-asm/src/parse/macros.rs | 2 - etk-asm/tests/asm.rs | 21 +- etk-asm/tests/asm/variable-push/included.etk | 594 +++++++++++++++++++ etk-asm/tests/asm/variable-push/main.etk | 7 + 6 files changed, 677 insertions(+), 66 deletions(-) create mode 100644 etk-asm/tests/asm/variable-push/included.etk create mode 100644 etk-asm/tests/asm/variable-push/main.etk diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 2b822289..5d2c0976 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -130,12 +130,11 @@ mod error { } pub use self::error::Error; -use crate::ast::Node; use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; use rand::Rng; use std::cmp; -use std::collections::{hash_map, HashMap, HashSet}; +use std::collections::{hash_map, HashMap}; /// An item to be assembled, which can be either an [`AbstractOp`], /// the inclusion of a new scope or a raw byte sequence. @@ -212,10 +211,6 @@ pub struct Assembler { /// Labels, in `pending`, that have been referred to (ex. with push) but /// have not been declared with an `AbstractOp::Label`. undefined_labels: Vec, - - /// Macros, in `pending`, that have been referred to but - /// have not been declared with an `AbstractOp::Macro`. - undefined_macros: Vec, } /// Struct used to keep track of pending label invocations and their positions in code. @@ -226,19 +221,9 @@ pub struct PendingLabel { /// Position where the label was invoked. position: usize, -} - -/// Struct used to keep track of pending macro invocations and their positions in code. -#[derive(Debug, Clone)] -pub struct PendingMacro { - /// The name of the macro. - name: String, - - /// Parameters of the macro. - parameters: Vec, - /// Position where the macro was invoked. - position: usize, + /// Whether the label was invoked with a dynamic push or not. + dynamic_push: bool, } impl Assembler { @@ -247,16 +232,16 @@ impl Assembler { Self::default() } - /// TODO: Add DOC - pub fn new_internal(concrete_len: usize) -> Self { + /// Create a new `Assembler` with a known length. + pub fn new_internal(actual_len: usize) -> Self { Self { - concrete_len, + concrete_len: actual_len, ..Self::default() } } - /// TODO: Add DOC - pub fn get_concrete_len(&self) -> usize { + /// Get the number of bytes that can be collected with [`Assembler::take`]. + pub fn concrete_len(&self) -> usize { self.concrete_len } @@ -340,13 +325,6 @@ impl Assembler { .fail(); } - /*if !self.undefined_macros.is_empty() { - return error::UndeclaredInstructionMacro { - name: self.undefined_macros[0].name.to_owned(), - } - .fail(); - }*/ - Ok(()) } @@ -388,16 +366,18 @@ impl Assembler { // Get all labels used by `rop`, check if they've been defined, and if not, note them as // "undeclared". - if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { + /*if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { for label in labels { if !self.declared_labels.contains_key(&label) { self.undefined_labels.push(PendingLabel { label: label.to_owned(), position: self.ready.len(), + //dynamic_push: false, + // TODO: Check dynamic push }); } } - } + }*/ Ok(()) } @@ -444,16 +424,18 @@ impl Assembler { let mut dst = 0; for ul in self.undefined_labels.iter() { if ul.label == label { - let tmp = ((self.concrete_len - ul.position) / 256) as f32; + // Compensation in case label was dynamically pushed. + let tmp = (self.concrete_len as f32 - ul.position as f32) / 256.0; dst = cmp::max(tmp.floor() as usize, dst); } } self.undefined_labels.retain(|l| l.label != *label); + self.concrete_len += dst; let old = self .declared_labels - .insert(label, Some(self.concrete_len + dst)) + .insert(label, Some(self.concrete_len)) .expect("label should exist"); assert_eq!(old, None, "label should have been undefined"); Ok(()) @@ -486,12 +468,21 @@ impl Assembler { } Err(ops::Error::ContextIncomplete { source }) => match source { UnknownLabel { label: _label, .. } => { - let size = op.size().unwrap_or(2); // %push(label) with min size (to be updated) - self.concrete_len += size; + let mut dynamic_push = false; + match op.size() { + Some(size) => self.concrete_len += size, + None => { + self.concrete_len += 2; + dynamic_push = true; + } + }; + //let size = op.size().unwrap_or(2); // %push(label) with min size (to be updated) + //self.concrete_len += size; self.undefined_labels.push(PendingLabel { label: _label.to_owned(), position: self.ready.len(), + dynamic_push, }); self.ready.push(rop); } @@ -509,14 +500,7 @@ impl Assembler { self.ready.push(RawOp::Raw(raw)); Ok(()) } - RawOp::Scope(ops) => { - let mut scope = Assembler::new_internal(self.concrete_len); - scope.inspect_macros(ops.iter().cloned())?; - let sz = scope.push_all(ops)?; - self.concrete_len += sz; - self.ready.push(RawOp::Scope(scope.ready)); - Ok(()) - } + RawOp::Scope(_) => unreachable!("scopes should be expanded before being pushed"), } } @@ -1218,7 +1202,7 @@ mod tests { Ok(()) } - #[test] + /*#[test] fn assemble_variable_push_expression_with_undeclared_labels() -> Result<(), Error> { let mut asm = Assembler::new(); asm.push_all(vec![ @@ -1232,7 +1216,7 @@ mod tests { let err = asm.finish().unwrap_err(); assert_matches!(err, Error::UndeclaredLabels { labels, .. } if (labels.contains(&"foo".to_string())) && labels.contains(&"bar".to_string())); Ok(()) - } + }*/ #[test] fn assemble_variable_push1_expression() -> Result<(), Error> { diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index d98a41b2..c362c0f6 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -113,7 +113,6 @@ struct Root { } impl Root { - /// TODO: Temporal DOC fn new(mut file: PathBuf) -> Result { // Pop the filename. if !file.pop() { @@ -199,8 +198,20 @@ impl Program { ensure!(self.depth <= 255, error::RecursionLimit); self.depth += 1; - let path = if let Some(ref root) = self.root { - let candidate = self.actual_path.join(path); + let oldpath = self.actual_path.clone(); + let new_path = if let Some(ref root) = self.root { + let candidate = match path.parent() { + Some(parent) => { + let mut candidate = self.actual_path.join(parent); + candidate.push(path.file_name().unwrap()); + candidate + } + None => { + let mut candidate = root.original.clone(); + candidate.push(path); + candidate + } + }; root.check(&candidate)?; candidate } else { @@ -208,7 +219,9 @@ impl Program { path }; - Ok(path) + self.actual_path = new_path; + + Ok(oldpath) } fn pop_path(&mut self, oldpath: PathBuf) { @@ -279,7 +292,8 @@ where Ok(()) } - /// TODO: Documentation + /// Assemble instructions from `src` as if they were read from a file located + /// at `path`. pub fn ingest

(&mut self, path: P, text: &str) -> Result<(), Error> where P: Into, @@ -299,12 +313,11 @@ where Ok(()) } - /// Assemble instructions from `src` as if they were read from a file located - /// at `path`. fn preprocess(&mut self, program: &mut Program, src: &str) -> Result, Error> { let nodes = parse_asm(&src)?; let mut raws = Vec::new(); for node in nodes { + println!("{:?}", node); match node { Node::Op(op) => { raws.push(RawOp::Op(op)); @@ -353,10 +366,11 @@ where { let path = path.into(); let oldpath = program.push_path(path.clone())?; - let code = read_to_string::<&PathBuf>(&path).with_context(|_| error::Io { - message: "reading file before parsing", - path: path.to_owned(), - })?; + let code = + read_to_string::<&PathBuf>(&program.actual_path).with_context(|_| error::Io { + message: "reading file before parsing", + path: path.to_owned(), + })?; let new_raws = self.preprocess(program, &code)?; program.pop_path(oldpath); Ok(new_raws) @@ -371,7 +385,7 @@ where asm.push(RawOp::Op(op))?; } RawOp::Scope(scope_ops) => { - let mut new_asm = Assembler::new_internal(asm.get_concrete_len()); + let mut new_asm = Assembler::new_internal(asm.concrete_len()); self.run(scope_ops, &mut new_asm)?; let raw = new_asm.take(); asm.push(RawOp::Raw(raw))?; @@ -456,7 +470,8 @@ mod tests { let mut output = Vec::new(); let mut ingest = Ingest::new(&mut output); ingest.ingest(root, &text)?; - assert_eq!(output, hex!("60015b586000566002")); + + assert_eq!(output, hex!("60015b586002566002")); Ok(()) } @@ -648,7 +663,7 @@ mod tests { let mut ingest = Ingest::new(&mut output); ingest.ingest(root, &text)?; - let expected = hex!("620000155b58600b5b5b61000b6100035b600360045b62000004"); + let expected = hex!("620000155b5860105b5b6100106100085b600860095b62000004"); assert_eq!(output, expected); Ok(()) diff --git a/etk-asm/src/parse/macros.rs b/etk-asm/src/parse/macros.rs index d3558f9b..c262917f 100644 --- a/etk-asm/src/parse/macros.rs +++ b/etk-asm/src/parse/macros.rs @@ -7,9 +7,7 @@ use crate::ops::{ AbstractOp, Expression, ExpressionMacroDefinition, ExpressionMacroInvocation, InstructionMacroDefinition, InstructionMacroInvocation, }; -use crate::parse::{error, parse_asm}; use pest::iterators::Pair; -use snafu::IntoError; use std::path::PathBuf; pub(crate) fn parse(pair: Pair) -> Result { diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 3b2d5463..a88ee250 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -47,7 +47,7 @@ fn out_of_bounds() { assert_matches!(err, Error::DirectoryTraversal { .. }); } -#[test] +/*#[test] fn subdirectory() -> Result<(), Error> { let mut output = Vec::new(); let mut ingester = Ingest::new(&mut output); @@ -56,8 +56,7 @@ fn subdirectory() -> Result<(), Error> { assert_eq!(output, hex!("63c001c0de60ff")); Ok(()) -} - +}*/ #[test] fn variable_jump() -> Result<(), Error> { let mut output = Vec::new(); @@ -293,6 +292,20 @@ fn every_op() -> Result<(), Error> { Ok(()) } +/*#[test] +fn test_dynamic_push_and_include() -> Result<(), Error> { + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + ingester.ingest_file(source(&["variable-push", "main.etk"]))?; + + //let str_out: Vec<_> = output.iter().map(|&byte| format!("{:02x}", byte)).collect(); + //println!("output: {:?}", str_out.concat()); + + assert_eq!(output, hex!("61025758585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585801010101010158585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858015b58")); + + Ok(()) +} + #[test] fn test_erc20() -> Result<(), Error> { let mut output = Vec::new(); @@ -303,4 +316,4 @@ fn test_erc20() -> Result<(), Error> { //assert_eq!(output, hex!("6000600060006000600060006000")); println!("output: {:?}", str_out.concat()); Ok(()) -} +}*/ diff --git a/etk-asm/tests/asm/variable-push/included.etk b/etk-asm/tests/asm/variable-push/included.etk new file mode 100644 index 00000000..c55517ab --- /dev/null +++ b/etk-asm/tests/asm/variable-push/included.etk @@ -0,0 +1,594 @@ +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +add +add +add +add +add +add +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +add diff --git a/etk-asm/tests/asm/variable-push/main.etk b/etk-asm/tests/asm/variable-push/main.etk new file mode 100644 index 00000000..cef9faf7 --- /dev/null +++ b/etk-asm/tests/asm/variable-push/main.etk @@ -0,0 +1,7 @@ +%push(label) +pc +pc +%include("included.etk") +label: +jumpdest +pc \ No newline at end of file From be7c9f0166dde4127d42860d24adcb1196e6e0a3 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Wed, 4 Oct 2023 11:21:12 -0300 Subject: [PATCH 22/48] Label insertion with built-in operation, fixed --- etk-asm/src/asm.rs | 12 +++++++++--- etk-asm/tests/asm.rs | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 5d2c0976..0ab05ffb 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -423,10 +423,16 @@ impl Assembler { RawOp::Op(AbstractOp::Label(label)) => { let mut dst = 0; for ul in self.undefined_labels.iter() { - if ul.label == label { + if (ul.label == label) & ul.dynamic_push { // Compensation in case label was dynamically pushed. - let tmp = (self.concrete_len as f32 - ul.position as f32) / 256.0; - dst = cmp::max(tmp.floor() as usize, dst); + let mut tmp = + ((self.concrete_len as f32 - ul.position as f32) / 256.0).floor(); + + // Size already accounted for %push(label) was 2. + if tmp >= 1.0 { + tmp -= 1.0; + } + dst = cmp::max(tmp as usize, dst); } } diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index a88ee250..4f9c87bb 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -292,7 +292,7 @@ fn every_op() -> Result<(), Error> { Ok(()) } -/*#[test] +#[test] fn test_dynamic_push_and_include() -> Result<(), Error> { let mut output = Vec::new(); let mut ingester = Ingest::new(&mut output); @@ -305,7 +305,7 @@ fn test_dynamic_push_and_include() -> Result<(), Error> { Ok(()) } - +/* #[test] fn test_erc20() -> Result<(), Error> { let mut output = Vec::new(); From 5ac947e38cbba4f802e4266e0bedec3e3d020da3 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Wed, 4 Oct 2023 12:03:12 -0300 Subject: [PATCH 23/48] Label insertion with built-in operation (minor fix) & Rustfmt modifications --- etk-asm/src/asm.rs | 46 +++++++++---------------------------------- etk-asm/src/ast.rs | 1 - etk-asm/src/ingest.rs | 7 ++----- 3 files changed, 11 insertions(+), 43 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 0ab05ffb..05b191bf 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -151,16 +151,6 @@ pub enum RawOp { Raw(Vec), } -impl RawOp { - fn expr(&self) -> Option<&Expression> { - match self { - Self::Op(op) => op.expr(), - Self::Scope(_) => None, - Self::Raw(_) => None, - } - } -} - impl From for RawOp { fn from(op: AbstractOp) -> Self { Self::Op(op) @@ -350,34 +340,16 @@ impl Assembler { O: Into, { let rop = rop.into(); - match rop { - RawOp::Op(AbstractOp::MacroDefinition(ref defn)) => { - match self.declared_macros.entry(defn.name().to_owned()) { - hash_map::Entry::Occupied(_) => { - return error::DuplicateMacro { name: defn.name() }.fail() - } - hash_map::Entry::Vacant(v) => { - v.insert(defn.to_owned()); - } + if let RawOp::Op(AbstractOp::MacroDefinition(ref defn)) = rop { + match self.declared_macros.entry(defn.name().to_owned()) { + hash_map::Entry::Occupied(_) => { + return error::DuplicateMacro { name: defn.name() }.fail() } - } - _ => {} - }; - - // Get all labels used by `rop`, check if they've been defined, and if not, note them as - // "undeclared". - /*if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { - for label in labels { - if !self.declared_labels.contains_key(&label) { - self.undefined_labels.push(PendingLabel { - label: label.to_owned(), - position: self.ready.len(), - //dynamic_push: false, - // TODO: Check dynamic push - }); + hash_map::Entry::Vacant(v) => { + v.insert(defn.to_owned()); } } - }*/ + } Ok(()) } @@ -429,7 +401,7 @@ impl Assembler { ((self.concrete_len as f32 - ul.position as f32) / 256.0).floor(); // Size already accounted for %push(label) was 2. - if tmp >= 1.0 { + if tmp > 1.0 { tmp -= 1.0; } dst = cmp::max(tmp as usize, dst); @@ -569,7 +541,7 @@ impl Assembler { Ok(Some(self.push_all(m.contents)?)) } - _ => return error::UndeclaredInstructionMacro { name }.fail(), + _ => error::UndeclaredInstructionMacro { name }.fail(), } } } diff --git a/etk-asm/src/ast.rs b/etk-asm/src/ast.rs index e32c93c0..e3824c15 100644 --- a/etk-asm/src/ast.rs +++ b/etk-asm/src/ast.rs @@ -6,7 +6,6 @@ use etk_ops::cancun::Op; #[derive(Debug, Clone, PartialEq)] pub(crate) enum Node { Op(AbstractOp), - Raw(Vec), Import(PathBuf), Include(PathBuf), IncludeHex(PathBuf), diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index c362c0f6..aec984a7 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -299,7 +299,7 @@ where P: Into, { let mut program = Program::new(Some(Root::new(path.into())?)); - let nodes = self.preprocess(&mut program, &text)?; + let nodes = self.preprocess(&mut program, text)?; let mut asm = Assembler::new(); self.run(nodes, &mut asm)?; @@ -314,7 +314,7 @@ where } fn preprocess(&mut self, program: &mut Program, src: &str) -> Result, Error> { - let nodes = parse_asm(&src)?; + let nodes = parse_asm(src)?; let mut raws = Vec::new(); for node in nodes { println!("{:?}", node); @@ -322,9 +322,6 @@ where Node::Op(op) => { raws.push(RawOp::Op(op)); } - Node::Raw(raw) => { - raws.push(RawOp::Raw(raw)); - } Node::Import(imp_path) => { let new_raws = self.resolve_and_ingest(program, imp_path)?; raws.extend(new_raws); From ce470578f6301e14a0a332fd4699bdd23aa37517 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Wed, 4 Oct 2023 14:01:57 -0300 Subject: [PATCH 24/48] Path resolution. More testing is needed --- etk-asm/src/ingest.rs | 58 +++++++++++++++++++------------------------ etk-asm/tests/asm.rs | 20 +++------------ 2 files changed, 28 insertions(+), 50 deletions(-) diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index aec984a7..1ce4afc7 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -182,15 +182,15 @@ impl Root { struct Program { depth: usize, root: Option, - actual_path: PathBuf, + sources: Vec, } impl Program { - fn new(root: Option) -> Self { + fn new(path: PathBuf) -> Self { Self { depth: 0, - root: root.clone(), - actual_path: root.map(|r| r.original).unwrap_or(PathBuf::new()), + root: Root::new(path.clone()).ok(), + sources: vec![path], } } @@ -198,35 +198,28 @@ impl Program { ensure!(self.depth <= 255, error::RecursionLimit); self.depth += 1; - let oldpath = self.actual_path.clone(); - let new_path = if let Some(ref root) = self.root { - let candidate = match path.parent() { - Some(parent) => { - let mut candidate = self.actual_path.join(parent); - candidate.push(path.file_name().unwrap()); - candidate - } - None => { - let mut candidate = root.original.clone(); - candidate.push(path); - candidate - } + let path = if let Some(ref root) = self.root { + let last = self.sources.last().unwrap(); + let dir = match last.parent() { + Some(s) => s, + None => Path::new("./"), }; + let candidate = dir.join(path); root.check(&candidate)?; + self.sources.push(candidate.clone()); candidate } else { + assert!(self.sources.is_empty()); self.root = Some(Root::new(path.clone())?); path }; - self.actual_path = new_path; - - Ok(oldpath) + Ok(path) } - fn pop_path(&mut self, oldpath: PathBuf) { + fn pop_path(&mut self) { self.depth -= 1; - self.actual_path = oldpath; + self.sources.pop(); } } @@ -298,10 +291,10 @@ where where P: Into, { - let mut program = Program::new(Some(Root::new(path.into())?)); + let mut program = Program::new(path.into()); let nodes = self.preprocess(&mut program, text)?; let mut asm = Assembler::new(); - self.run(nodes, &mut asm)?; + Self::run(nodes, &mut asm)?; let raw = asm.take(); @@ -362,18 +355,17 @@ where P: Into, { let path = path.into(); - let oldpath = program.push_path(path.clone())?; - let code = - read_to_string::<&PathBuf>(&program.actual_path).with_context(|_| error::Io { - message: "reading file before parsing", - path: path.to_owned(), - })?; + let source = program.push_path(path.clone())?; + let code = read_to_string::<&PathBuf>(&source).with_context(|_| error::Io { + message: "reading file before parsing", + path: path.to_owned(), + })?; let new_raws = self.preprocess(program, &code)?; - program.pop_path(oldpath); + program.pop_path(); Ok(new_raws) } - fn run(&mut self, ops: Vec, asm: &mut Assembler) -> Result<(), Error> { + fn run(ops: Vec, asm: &mut Assembler) -> Result<(), Error> { asm.inspect_macros(ops.clone())?; for rawop in ops { @@ -383,7 +375,7 @@ where } RawOp::Scope(scope_ops) => { let mut new_asm = Assembler::new_internal(asm.concrete_len()); - self.run(scope_ops, &mut new_asm)?; + Self::run(scope_ops, &mut new_asm)?; let raw = new_asm.take(); asm.push(RawOp::Raw(raw))?; } diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 4f9c87bb..c09dba9f 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -47,7 +47,7 @@ fn out_of_bounds() { assert_matches!(err, Error::DirectoryTraversal { .. }); } -/*#[test] +#[test] fn subdirectory() -> Result<(), Error> { let mut output = Vec::new(); let mut ingester = Ingest::new(&mut output); @@ -56,7 +56,8 @@ fn subdirectory() -> Result<(), Error> { assert_eq!(output, hex!("63c001c0de60ff")); Ok(()) -}*/ +} + #[test] fn variable_jump() -> Result<(), Error> { let mut output = Vec::new(); @@ -298,22 +299,7 @@ fn test_dynamic_push_and_include() -> Result<(), Error> { let mut ingester = Ingest::new(&mut output); ingester.ingest_file(source(&["variable-push", "main.etk"]))?; - //let str_out: Vec<_> = output.iter().map(|&byte| format!("{:02x}", byte)).collect(); - //println!("output: {:?}", str_out.concat()); - assert_eq!(output, hex!("61025758585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585801010101010158585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858015b58")); Ok(()) } -/* -#[test] -fn test_erc20() -> Result<(), Error> { - let mut output = Vec::new(); - let mut ingester = Ingest::new(&mut output); - ingester.ingest_file(source(&["erc20", "Contract.etk"]))?; - - let str_out: Vec<_> = output.iter().map(|&byte| format!("{:02x}", byte)).collect(); - //assert_eq!(output, hex!("6000600060006000600060006000")); - println!("output: {:?}", str_out.concat()); - Ok(()) -}*/ From e6ff678cc1f5abb21f8daee98102dba9d5ae6215 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 5 Oct 2023 11:15:41 -0300 Subject: [PATCH 25/48] Test, code cleaning, etc --- etk-asm/src/asm.rs | 52 +++++++++++++++++++++++++++++++++++-------- etk-asm/src/ingest.rs | 1 - 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 05b191bf..4ec6d2a5 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -126,6 +126,17 @@ mod error { #[snafu(backtrace)] source: ParseError, }, + + /// An instruction macro was used without being defined. + #[snafu(display("variable {} inside macro, was never defined", var))] + #[non_exhaustive] + UndeclaredVariableMacro { + /// The variable that was used without being defined. + var: String, + + /// The location of the error. + backtrace: Backtrace, + }, } } @@ -235,7 +246,7 @@ impl Assembler { self.concrete_len } - /// TODO: Add DOC + /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. pub fn inspect_macros(&mut self, nodes: I) -> Result<(), Error> where I: IntoIterator, @@ -289,7 +300,9 @@ impl Assembler { return error::UndeclaredInstructionMacro { name }.fail(); } - UndefinedVariable { name, .. } => todo!("undefined variable {}", name), + UndefinedVariable { name, .. } => { + return error::UndeclaredVariableMacro { var: name }.fail(); + } }, Err(_) => unreachable!("all ops should be concretizable"), } @@ -362,7 +375,6 @@ impl Assembler { O: Into, { let rop = rop.into(); - println!("pushing {:?}", rop); self.declare_label(&rop)?; @@ -454,8 +466,6 @@ impl Assembler { dynamic_push = true; } }; - //let size = op.size().unwrap_or(2); // %push(label) with min size (to be updated) - //self.concrete_len += size; self.undefined_labels.push(PendingLabel { label: _label.to_owned(), @@ -467,7 +477,9 @@ impl Assembler { UnknownMacro { name, .. } => { return error::UndeclaredInstructionMacro { name }.fail() } - UndefinedVariable { name, .. } => todo!("undefined variable {}", name), + UndefinedVariable { name, .. } => { + return error::UndeclaredVariableMacro { var: name }.fail() + } }, } @@ -1023,8 +1035,6 @@ mod tests { let mut asm = Assembler::new(); asm.inspect_macros(ops.clone())?; let err = asm.push_all(ops).unwrap_err(); - //let err = asm.finish().unwrap_err(); - println!("{:?}", err); assert_matches!(err, Error::UndeclaredInstructionMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1048,7 +1058,6 @@ mod tests { ]; let mut asm = Assembler::new(); let err = asm.inspect_macros(ops.clone()).unwrap_err(); - //let err = asm.push_all(ops).unwrap_err(); assert_matches!(err, Error::DuplicateMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1254,4 +1263,29 @@ mod tests { Ok(()) } + + #[test] + fn assemble_instruction_macro_with_undeclared_variables() { + let ops = vec![ + InstructionMacroDefinition { + name: "my_macro".into(), + parameters: vec!["foo".into()], + contents: vec![AbstractOp::new(Push1(Imm::with_variable("bar")))], + } + .into(), + AbstractOp::Label("b".into()), + AbstractOp::new(JumpDest), + AbstractOp::new(Push1(Imm::with_label("b"))), + AbstractOp::Macro(InstructionMacroInvocation { + name: "my_macro".into(), + parameters: vec![BigInt::from_bytes_be(Sign::Plus, &vec![0x42]).into()], + }), + ]; + + let mut asm = Assembler::new(); + asm.inspect_macros(ops.clone()).unwrap(); + let err = asm.push_all(ops).unwrap_err(); + + assert_matches!(err, Error::UndeclaredVariableMacro { var, .. } if var == "bar"); + } } diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index 1ce4afc7..b0281894 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -310,7 +310,6 @@ where let nodes = parse_asm(src)?; let mut raws = Vec::new(); for node in nodes { - println!("{:?}", node); match node { Node::Op(op) => { raws.push(RawOp::Op(op)); From 1f4696581f0398421de24bf57c92cbcce1e38347 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 5 Oct 2023 11:40:33 -0300 Subject: [PATCH 26/48] Test with short-circuit evaluation --- etk-asm/src/asm.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 4ec6d2a5..d36009c7 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -1189,7 +1189,7 @@ mod tests { Ok(()) } - /*#[test] + #[test] fn assemble_variable_push_expression_with_undeclared_labels() -> Result<(), Error> { let mut asm = Assembler::new(); asm.push_all(vec![ @@ -1201,9 +1201,10 @@ mod tests { AbstractOp::new(Gas), ])?; let err = asm.finish().unwrap_err(); - assert_matches!(err, Error::UndeclaredLabels { labels, .. } if (labels.contains(&"foo".to_string())) && labels.contains(&"bar".to_string())); + // The expressions have short-circuit evaluation, so only the first label is caught in the error. + assert_matches!(err, Error::UndeclaredLabels { labels, .. } if (labels.contains(&"foo".to_string()))); Ok(()) - }*/ + } #[test] fn assemble_variable_push1_expression() -> Result<(), Error> { From 2c6861158ffb1390ddced2c3c6bc57ada9171827 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 6 Oct 2023 10:56:34 -0300 Subject: [PATCH 27/48] Undoing of the modifications to the expected values of the tests. Addressing some of the changes proposed by Sam. --- etk-asm/src/asm.rs | 127 ++++++++++++++++++++---------------------- etk-asm/src/ingest.rs | 6 +- 2 files changed, 62 insertions(+), 71 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index d36009c7..321d0d68 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -128,7 +128,7 @@ mod error { }, /// An instruction macro was used without being defined. - #[snafu(display("variable {} inside macro, was never defined", var))] + #[snafu(display("variable `{}` inside macro, was never defined", var))] #[non_exhaustive] UndeclaredVariableMacro { /// The variable that was used without being defined. @@ -197,26 +197,26 @@ impl From> for RawOp { /// ``` #[derive(Debug, Default)] pub struct Assembler { - /// Assembled ops, not yet ready to be taken. + /// Assembled ops. ready: Vec, - /// Number of bytes in 'ready' that are ready to be taken. + /// Number of bytes used by the operations in ready. concrete_len: usize, - /// Labels, in `pending`, associated with an `AbstractOp::Label`. + /// Labels associated with an `AbstractOp::Label`. declared_labels: HashMap>, - /// Macros, in `pending`, associated with an `AbstractOp::Macro`. + /// Macros associated with an `AbstractOp::Macro`. declared_macros: HashMap, - /// Labels, in `pending`, that have been referred to (ex. with push) but + /// Labels that have been referred to (ex. with push) but /// have not been declared with an `AbstractOp::Label`. - undefined_labels: Vec, + undeclared_labels: Vec, } /// Struct used to keep track of pending label invocations and their positions in code. #[derive(Debug, Clone)] -pub struct PendingLabel { +struct PendingLabel { /// The name of the label. label: String, @@ -233,19 +233,6 @@ impl Assembler { Self::default() } - /// Create a new `Assembler` with a known length. - pub fn new_internal(actual_len: usize) -> Self { - Self { - concrete_len: actual_len, - ..Self::default() - } - } - - /// Get the number of bytes that can be collected with [`Assembler::take`]. - pub fn concrete_len(&self) -> usize { - self.concrete_len - } - /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. pub fn inspect_macros(&mut self, nodes: I) -> Result<(), Error> where @@ -284,26 +271,30 @@ impl Assembler { .concretize((&self.declared_labels, &self.declared_macros).into()) { Ok(cop) => cop.assemble(&mut output), - Err(ops::Error::ContextIncomplete { source }) => match source { - UnknownLabel { label: _label, .. } => { - let undefined_names: Vec<_> = self - .undefined_labels - .iter() - .map(|PendingLabel { label, .. }| label.clone()) - .collect(); - return error::UndeclaredLabels { - labels: undefined_names, - } - .fail(); - } - UnknownMacro { name, .. } => { - return error::UndeclaredInstructionMacro { name }.fail(); + Err(ops::Error::ContextIncomplete { + source: UnknownLabel { label: _label, .. }, + }) => { + let undefined_names: Vec<_> = self + .undeclared_labels + .iter() + .map(|PendingLabel { label, .. }| label.clone()) + .collect(); + return error::UndeclaredLabels { + labels: undefined_names, } + .fail(); + } + Err(ops::Error::ContextIncomplete { + source: UnknownMacro { name, .. }, + }) => { + return error::UndeclaredInstructionMacro { name }.fail(); + } + Err(ops::Error::ContextIncomplete { + source: UndefinedVariable { name, .. }, + }) => { + return error::UndeclaredVariableMacro { var: name }.fail(); + } - UndefinedVariable { name, .. } => { - return error::UndeclaredVariableMacro { var: name }.fail(); - } - }, Err(_) => unreachable!("all ops should be concretizable"), } } else if let RawOp::Raw(raw) = op { @@ -317,10 +308,10 @@ impl Assembler { /// Indicate that the input sequence is complete. Returns any errors that /// may remain. pub fn finish(&mut self) -> Result<(), Error> { - if !self.undefined_labels.is_empty() { + if !self.undeclared_labels.is_empty() { return error::UndeclaredLabels { labels: self - .undefined_labels + .undeclared_labels .iter() .map(|l| l.label.to_owned()) .collect::>(), @@ -406,7 +397,7 @@ impl Assembler { match rop { RawOp::Op(AbstractOp::Label(label)) => { let mut dst = 0; - for ul in self.undefined_labels.iter() { + for ul in self.undeclared_labels.iter() { if (ul.label == label) & ul.dynamic_push { // Compensation in case label was dynamically pushed. let mut tmp = @@ -420,7 +411,7 @@ impl Assembler { } } - self.undefined_labels.retain(|l| l.label != *label); + self.undeclared_labels.retain(|l| l.label != *label); self.concrete_len += dst; let old = self @@ -456,31 +447,31 @@ impl Assembler { } .fail() } - Err(ops::Error::ContextIncomplete { source }) => match source { - UnknownLabel { label: _label, .. } => { - let mut dynamic_push = false; - match op.size() { - Some(size) => self.concrete_len += size, - None => { - self.concrete_len += 2; - dynamic_push = true; - } - }; - - self.undefined_labels.push(PendingLabel { - label: _label.to_owned(), - position: self.ready.len(), - dynamic_push, - }); - self.ready.push(rop); - } - UnknownMacro { name, .. } => { - return error::UndeclaredInstructionMacro { name }.fail() - } - UndefinedVariable { name, .. } => { - return error::UndeclaredVariableMacro { var: name }.fail() - } - }, + Err(ops::Error::ContextIncomplete { + source: UnknownLabel { label: _label, .. }, + }) => { + let mut dynamic_push = false; + match op.size() { + Some(size) => self.concrete_len += size, + None => { + self.concrete_len += 2; + dynamic_push = true; + } + }; + + self.undeclared_labels.push(PendingLabel { + label: _label.to_owned(), + position: self.ready.len(), + dynamic_push, + }); + self.ready.push(rop); + } + Err(ops::Error::ContextIncomplete { + source: UnknownMacro { name, .. }, + }) => return error::UndeclaredInstructionMacro { name }.fail(), + Err(ops::Error::ContextIncomplete { + source: UndefinedVariable { name, .. }, + }) => return error::UndeclaredVariableMacro { var: name }.fail(), } Ok(()) diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index b0281894..72b8284b 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -373,7 +373,7 @@ where asm.push(RawOp::Op(op))?; } RawOp::Scope(scope_ops) => { - let mut new_asm = Assembler::new_internal(asm.concrete_len()); + let mut new_asm = Assembler::new(); Self::run(scope_ops, &mut new_asm)?; let raw = new_asm.take(); asm.push(RawOp::Raw(raw))?; @@ -459,7 +459,7 @@ mod tests { let mut ingest = Ingest::new(&mut output); ingest.ingest(root, &text)?; - assert_eq!(output, hex!("60015b586002566002")); + assert_eq!(output, hex!("60015b586000566002")); Ok(()) } @@ -651,7 +651,7 @@ mod tests { let mut ingest = Ingest::new(&mut output); ingest.ingest(root, &text)?; - let expected = hex!("620000155b5860105b5b6100106100085b600860095b62000004"); + let expected = hex!("620000155b58600b5b5b61000b6100035b600360045b62000004"); assert_eq!(output, expected); Ok(()) From ee041572eed2965509cbb121ba06f1a76f71f76e Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 6 Oct 2023 13:38:48 -0300 Subject: [PATCH 28/48] Code restructuring. push_all replaced by assemble --- etk-asm/src/asm.rs | 274 ++++++++++++++++++------------------------ etk-asm/src/ingest.rs | 29 +---- 2 files changed, 119 insertions(+), 184 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 321d0d68..275bb9d1 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -187,12 +187,10 @@ impl From> for RawOp { /// # /// # use hex_literal::hex; /// let mut asm = Assembler::new(); -/// asm.push_all(vec![ +/// let result = asm.assemble(vec![ /// AbstractOp::new(GetPc), /// ])?; -/// let output = asm.take(); -/// asm.finish()?; -/// # assert_eq!(output, hex!("58")); +/// # assert_eq!(result, hex!("58")); /// # Result::<(), Error>::Ok(()) /// ``` #[derive(Debug, Default)] @@ -234,12 +232,12 @@ impl Assembler { } /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. - pub fn inspect_macros(&mut self, nodes: I) -> Result<(), Error> + fn inspect_macros(&mut self, nodes: &I) -> Result<(), Error> where - I: IntoIterator, + I: IntoIterator + Clone, O: Into, { - for op in nodes { + for op in nodes.clone() { let op = op.into(); if let RawOp::Op(AbstractOp::MacroDefinition(_)) = op { self.declare_macro(op)? @@ -250,7 +248,7 @@ impl Assembler { } /// Collect any assembled instructions that are ready to be output. - pub fn take(&mut self) -> Vec { + fn take(&mut self) -> Vec { let output = self.concretize_ops(); match output { Ok(v) => { @@ -274,13 +272,13 @@ impl Assembler { Err(ops::Error::ContextIncomplete { source: UnknownLabel { label: _label, .. }, }) => { - let undefined_names: Vec<_> = self + let undeclared_names: Vec<_> = self .undeclared_labels .iter() .map(|PendingLabel { label, .. }| label.clone()) .collect(); return error::UndeclaredLabels { - labels: undefined_names, + labels: undeclared_names, } .fail(); } @@ -305,9 +303,9 @@ impl Assembler { Ok(output) } - /// Indicate that the input sequence is complete. Returns any errors that + /// Check if the input sequence is complete. Returns any errors that /// may remain. - pub fn finish(&mut self) -> Result<(), Error> { + fn finish(&mut self) -> Result<(), Error> { if !self.undeclared_labels.is_empty() { return error::UndeclaredLabels { labels: self @@ -324,21 +322,24 @@ impl Assembler { /// Feed instructions into the `Assembler`. /// - /// Returns the number of bytes that can be collected with [`Assembler::take`]. - pub fn push_all(&mut self, ops: I) -> Result + /// Returns the code of the assembled program. + pub fn assemble(&mut self, ops: I) -> Result, Error> where - I: IntoIterator, + I: IntoIterator + Clone, O: Into, { + self.inspect_macros(&ops)?; + for op in ops { self.push(op)?; } - Ok(self.concrete_len) + self.finish()?; + + Ok(self.take()) } - /// Insert explicilty declared macros and labels, via `AbstractOp`, and implictly declared - /// macros and labels via usage in `Op`. + /// Insert explicitly declared macros, via `AbstractOp`, into the `Assembler`. fn declare_macro(&mut self, rop: O) -> Result<(), Error> where O: Into, @@ -361,7 +362,7 @@ impl Assembler { /// Feed a single instruction into the `Assembler`. /// /// Returns the number of bytes that can be collected with [`Assembler::take`] - pub fn push(&mut self, rop: O) -> Result + fn push(&mut self, rop: O) -> Result where O: Into, { @@ -481,7 +482,13 @@ impl Assembler { self.ready.push(RawOp::Raw(raw)); Ok(()) } - RawOp::Scope(_) => unreachable!("scopes should be expanded before being pushed"), + RawOp::Scope(scope) => { + let mut asm = Self::new(); + let scope_result = asm.assemble(scope)?; + self.concrete_len += scope_result.len(); + self.ready.push(RawOp::Raw(scope_result)); + Ok(()) + } } } @@ -542,7 +549,10 @@ impl Assembler { } } - Ok(Some(self.push_all(m.contents)?)) + for op in m.contents.iter() { + self.push(op.clone())?; + } + Ok(Some(self.concrete_len)) } _ => error::UndeclaredInstructionMacro { name }.fail(), } @@ -564,20 +574,19 @@ mod tests { #[test] fn assemble_variable_push_const_while_pending() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::Op(Push1(Imm::with_label("label1")).into()), AbstractOp::Push(Terminal::Number(0xaabb.into()).into()), AbstractOp::Label("label1".into()), ])?; - assert_eq!(5, sz); - assert_eq!(asm.take(), hex!("600561aabb")); + assert_eq!(result, hex!("600561aabb")); Ok(()) } #[test] fn assemble_variable_pushes_abab() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("label1")), AbstractOp::Push(Imm::with_label("label2")), @@ -586,15 +595,14 @@ mod tests { AbstractOp::Label("label2".into()), AbstractOp::new(GetPc), ])?; - assert_eq!(7, sz); - assert_eq!(asm.take(), hex!("5b600560065858")); + assert_eq!(result, hex!("5b600560065858")); Ok(()) } #[test] fn assemble_variable_pushes_abba() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("label1")), AbstractOp::Push(Imm::with_label("label2")), @@ -603,33 +611,30 @@ mod tests { AbstractOp::Label("label1".into()), AbstractOp::new(GetPc), ])?; - assert_eq!(7, sz); - assert_eq!(asm.take(), hex!("5b600660055858")); + assert_eq!(result, hex!("5b600660055858")); Ok(()) } #[test] fn assemble_variable_push1_multiple() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), ])?; - assert_eq!(5, sz); - assert_eq!(asm.take(), hex!("5b60056005")); + assert_eq!(result, hex!("5b60056005")); Ok(()) } #[test] fn assemble_variable_push_const() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![AbstractOp::Push( + let result = asm.assemble(vec![AbstractOp::Push( Terminal::Number((0x00aaaaaaaaaaaaaaaaaaaaaaaa as u128).into()).into(), )])?; - assert_eq!(13, sz); - assert_eq!(asm.take(), hex!("6baaaaaaaaaaaaaaaaaaaaaaaa")); + assert_eq!(result, hex!("6baaaaaaaaaaaaaaaaaaaaaaaa")); Ok(()) } @@ -639,7 +644,7 @@ mod tests { let mut asm = Assembler::new(); let err = asm - .push_all(vec![AbstractOp::Push(Terminal::Number(v).into())]) + .assemble(vec![AbstractOp::Push(Terminal::Number(v).into())]) .unwrap_err(); assert_matches!(err, Error::ExpressionTooLarge { .. }); @@ -649,7 +654,7 @@ mod tests { fn assemble_variable_push_negative() { let mut asm = Assembler::new(); let err = asm - .push_all(vec![AbstractOp::Push(Terminal::Number((-1).into()).into())]) + .assemble(vec![AbstractOp::Push(Terminal::Number((-1).into()).into())]) .unwrap_err(); assert_matches!(err, Error::ExpressionNegative { .. }); @@ -658,73 +663,68 @@ mod tests { #[test] fn assemble_variable_push_const0() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![AbstractOp::Push( + let result = asm.assemble(vec![AbstractOp::Push( Terminal::Number((0x00 as u128).into()).into(), )])?; - assert_eq!(2, sz); - assert_eq!(asm.take(), hex!("6000")); + assert_eq!(result, hex!("6000")); Ok(()) } #[test] fn assemble_variable_push1_known() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Label("auto".into()), AbstractOp::Push(Imm::with_label("auto")), ])?; - assert_eq!(3, sz); - assert_eq!(asm.take(), hex!("5b6001")); + assert_eq!(result, hex!("5b6001")); Ok(()) } #[test] fn assemble_variable_push1() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), AbstractOp::new(JumpDest), ])?; - assert_eq!(3, sz); - assert_eq!(asm.take(), hex!("60025b")); + assert_eq!(result, hex!("60025b")); Ok(()) } #[test] fn assemble_variable_push1_reuse() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), AbstractOp::new(JumpDest), AbstractOp::new(Push1(Imm::with_label("auto"))), ])?; - assert_eq!(5, sz); - assert_eq!(asm.take(), hex!("60025b6002")); + assert_eq!(result, hex!("60025b6002")); Ok(()) } #[test] fn assemble_variable_push2() -> Result<(), Error> { - let mut asm = Assembler::new(); - asm.push(AbstractOp::Push(Imm::with_label("auto")))?; + let mut code = vec![]; + code.push(AbstractOp::Push(Imm::with_label("auto"))); for _ in 0..255 { - asm.push(AbstractOp::new(GetPc))?; + code.push(AbstractOp::new(GetPc)); } - asm.push_all(vec![ - AbstractOp::Label("auto".into()), - AbstractOp::new(JumpDest), - ])?; + code.push(AbstractOp::Label("auto".into())); + code.push(AbstractOp::new(JumpDest)); + + let mut asm = Assembler::new(); + let result = asm.assemble(code)?; let mut expected = vec![0x61, 0x01, 0x02]; expected.extend_from_slice(&[0x58; 255]); expected.push(0x5b); - assert_eq!(asm.take(), expected); - - asm.finish()?; + assert_eq!(result, expected); Ok(()) } @@ -732,8 +732,9 @@ mod tests { #[test] fn assemble_undeclared_label() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push_all(vec![AbstractOp::new(Push1(Imm::with_label("hi")))])?; - let err = asm.finish().unwrap_err(); + let err = asm + .assemble(vec![AbstractOp::new(Push1(Imm::with_label("hi")))]) + .unwrap_err(); assert_matches!(err, Error::UndeclaredLabels { labels, .. } if labels == vec!["hi"]); Ok(()) } @@ -741,10 +742,9 @@ mod tests { #[test] fn assemble_jumpdest_no_label() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![AbstractOp::new(JumpDest)])?; - assert_eq!(1, sz); + let result = asm.assemble(vec![AbstractOp::new(JumpDest)])?; assert!(asm.declared_labels.is_empty()); - assert_eq!(asm.take(), hex!("5b")); + assert_eq!(result, hex!("5b")); Ok(()) } @@ -753,11 +753,10 @@ mod tests { let mut asm = Assembler::new(); let ops = vec![AbstractOp::Label("lbl".into()), AbstractOp::new(JumpDest)]; - let sz = asm.push_all(ops)?; - assert_eq!(1, sz); + let result = asm.assemble(ops)?; assert_eq!(asm.declared_labels.len(), 1); assert_eq!(asm.declared_labels.get("lbl"), Some(&Some(0))); - assert_eq!(asm.take(), hex!("5b")); + assert_eq!(result, hex!("5b")); Ok(()) } @@ -770,9 +769,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 3); - assert_eq!(asm.take(), hex!("5b6000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b6000")); Ok(()) } @@ -786,9 +784,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 3); - assert_eq!(asm.take(), hex!("600258")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("600258")); Ok(()) } @@ -802,9 +799,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 3); - assert_eq!(asm.take(), hex!("60025b")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("60025b")); Ok(()) } @@ -818,7 +814,7 @@ mod tests { ops.push(AbstractOp::new(JumpDest)); ops.push(AbstractOp::new(Push1(Imm::with_label("a")))); let mut asm = Assembler::new(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::ExpressionTooLarge { expr: Expression::Terminal(Terminal::Label(label)), .. } if label == "a"); } @@ -831,11 +827,7 @@ mod tests { ops.push(AbstractOp::new(JumpDest)); ops.push(AbstractOp::new(Push1(Imm::with_label("b")))); let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 259); - - let assembled = asm.take(); - asm.finish()?; + let result = asm.assemble(ops)?; let mut expected = vec![0x58; 255]; expected.push(0x5b); @@ -843,7 +835,7 @@ mod tests { expected.push(0x60); expected.push(0xff); - assert_eq!(assembled, expected); + assert_eq!(result, expected); Ok(()) } @@ -874,11 +866,8 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(sz, 0); - let out = asm.take(); - assert_eq!(out, []); + let result = asm.assemble(ops)?; + assert_eq!(result, []); Ok(()) } @@ -911,11 +900,8 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(sz, 13); - let out = asm.take(); - assert_eq!(out, hex!("5b60005b600360005b60086000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b60005b600360005b60086000")); Ok(()) } @@ -944,11 +930,8 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(sz, 8); - let out = asm.take(); - assert_eq!(out, hex!("5b60005b60036000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b60005b60036000")); Ok(()) } @@ -977,11 +960,8 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(sz, 8); - let out = asm.take(); - assert_eq!(out, hex!("5b60005b60036000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b60005b60036000")); Ok(()) } @@ -1010,10 +990,8 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(7, sz); - assert_eq!(asm.take(), hex!("5b600560065858")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b600560065858")); Ok(()) } @@ -1024,8 +1002,7 @@ mod tests { InstructionMacroInvocation::with_zero_parameters("my_macro".into()), )]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::UndeclaredInstructionMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1048,7 +1025,7 @@ mod tests { .into(), ]; let mut asm = Assembler::new(); - let err = asm.inspect_macros(ops.clone()).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::DuplicateMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1068,8 +1045,7 @@ mod tests { )), ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::DuplicateLabel { label, .. } if label == "a"); Ok(()) @@ -1096,14 +1072,9 @@ mod tests { AbstractOp::new(Push1(Imm::with_label("a"))), ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(sz, 5); - - let out = asm.take(); - asm.finish()?; + let result = asm.assemble(ops)?; - assert_eq!(out, hex!("3360016000")); + assert_eq!(result, hex!("3360016000")); Ok(()) } @@ -1133,11 +1104,8 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(sz, 7); - let out = asm.take(); - assert_eq!(out, hex!("5b600060426000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b600060426000")); Ok(()) } @@ -1149,10 +1117,8 @@ mod tests { )))]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 2); - let out = asm.take(); - assert_eq!(out, hex!("6002")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("6002")); Ok(()) } @@ -1163,7 +1129,7 @@ mod tests { BigInt::from(-1).into(), )))]; let mut asm = Assembler::new(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::ExpressionNegative { value, .. } if value == BigInt::from(-1)); Ok(()) @@ -1172,10 +1138,11 @@ mod tests { #[test] fn assemble_expression_undeclared_label() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push_all(vec![AbstractOp::new(Push1(Imm::with_expression( - Terminal::Label(String::from("hi")).into(), - )))])?; - let err = asm.finish().unwrap_err(); + let err = asm + .assemble(vec![AbstractOp::new(Push1(Imm::with_expression( + Terminal::Label(String::from("hi")).into(), + )))]) + .unwrap_err(); assert_matches!(err, Error::UndeclaredLabels { labels, .. } if labels == vec!["hi"]); Ok(()) } @@ -1183,15 +1150,16 @@ mod tests { #[test] fn assemble_variable_push_expression_with_undeclared_labels() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push_all(vec![ - AbstractOp::new(JumpDest), - AbstractOp::Push(Imm::with_expression(Expression::Plus( - Terminal::Label("foo".into()).into(), - Terminal::Label("bar".into()).into(), - ))), - AbstractOp::new(Gas), - ])?; - let err = asm.finish().unwrap_err(); + let err = asm + .assemble(vec![ + AbstractOp::new(JumpDest), + AbstractOp::Push(Imm::with_expression(Expression::Plus( + Terminal::Label("foo".into()).into(), + Terminal::Label("bar".into()).into(), + ))), + AbstractOp::new(Gas), + ]) + .unwrap_err(); // The expressions have short-circuit evaluation, so only the first label is caught in the error. assert_matches!(err, Error::UndeclaredLabels { labels, .. } if (labels.contains(&"foo".to_string()))); Ok(()) @@ -1200,7 +1168,7 @@ mod tests { #[test] fn assemble_variable_push1_expression() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Label("auto".into()), AbstractOp::Push(Imm::with_expression(Expression::Plus( @@ -1208,15 +1176,14 @@ mod tests { Terminal::Label(String::from("auto")).into(), ))), ])?; - assert_eq!(3, sz); - assert_eq!(asm.take(), hex!("5b6002")); + assert_eq!(result, hex!("5b6002")); Ok(()) } #[test] fn assemble_expression_with_labels() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_expression(Expression::Plus( Terminal::Label(String::from("foo")).into(), @@ -1226,8 +1193,7 @@ mod tests { AbstractOp::Label("foo".into()), AbstractOp::Label("bar".into()), ])?; - assert_eq!(4, sz); - assert_eq!(asm.take(), hex!("5b60085a")); + assert_eq!(result, hex!("5b60085a")); Ok(()) } @@ -1247,11 +1213,8 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone())?; - let sz = asm.push_all(ops)?; - assert_eq!(sz, 2); - let out = asm.take(); - assert_eq!(out, hex!("6002")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("6002")); Ok(()) } @@ -1275,8 +1238,7 @@ mod tests { ]; let mut asm = Assembler::new(); - asm.inspect_macros(ops.clone()).unwrap(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::UndeclaredVariableMacro { var, .. } if var == "bar"); } diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index 72b8284b..bc5e4bfd 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -294,9 +294,7 @@ where let mut program = Program::new(path.into()); let nodes = self.preprocess(&mut program, text)?; let mut asm = Assembler::new(); - Self::run(nodes, &mut asm)?; - - let raw = asm.take(); + let raw = asm.assemble(nodes)?; self.output.write_all(&raw).context(error::Io { message: "writing output", @@ -363,31 +361,6 @@ where program.pop_path(); Ok(new_raws) } - - fn run(ops: Vec, asm: &mut Assembler) -> Result<(), Error> { - asm.inspect_macros(ops.clone())?; - - for rawop in ops { - match rawop { - RawOp::Op(op) => { - asm.push(RawOp::Op(op))?; - } - RawOp::Scope(scope_ops) => { - let mut new_asm = Assembler::new(); - Self::run(scope_ops, &mut new_asm)?; - let raw = new_asm.take(); - asm.push(RawOp::Raw(raw))?; - } - RawOp::Raw(hex) => { - asm.push(RawOp::Raw(hex))?; - } - } - } - - asm.finish()?; - - Ok(()) - } } #[cfg(test)] From 859e85b35eec8d028ce4128905a50c3176d3d328 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 6 Oct 2023 14:04:18 -0300 Subject: [PATCH 29/48] Minor changes in docs --- etk-asm/src/asm.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 275bb9d1..5ff555f3 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -154,7 +154,7 @@ pub enum RawOp { /// An instruction to be assembled. Op(AbstractOp), - /// A new scope to be included. + /// A new scope to be created with its corresponding list of operations. Scope(Vec), /// Raw bytes, for example from `%include_hex`, to be included verbatim in @@ -198,7 +198,7 @@ pub struct Assembler { /// Assembled ops. ready: Vec, - /// Number of bytes used by the operations in ready. + /// Number of bytes used by the operations in `ready``. concrete_len: usize, /// Labels associated with an `AbstractOp::Label`. @@ -360,8 +360,6 @@ impl Assembler { } /// Feed a single instruction into the `Assembler`. - /// - /// Returns the number of bytes that can be collected with [`Assembler::take`] fn push(&mut self, rop: O) -> Result where O: Into, From 7bbfe8e9ab77fa278f54645841aec0efdb8c3914 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 6 Oct 2023 16:13:42 -0300 Subject: [PATCH 30/48] Added tests mentioned in issue #108 --- etk-asm/src/asm.rs | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 5ff555f3..3e5556db 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -1240,4 +1240,70 @@ mod tests { assert_matches!(err, Error::UndeclaredVariableMacro { var, .. } if var == "bar"); } + + #[test] + fn assemble_instruction_macro_two_delayed_definitions_mirrored() -> Result<(), Error> { + let ops = vec![ + AbstractOp::new(GetPc), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro1".into(), + parameters: vec![], + }), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro0".into(), + parameters: vec![], + }), + InstructionMacroDefinition { + name: "macro0".into(), + parameters: vec![], + contents: vec![AbstractOp::new(JumpDest)], + } + .into(), + InstructionMacroDefinition { + name: "macro1".into(), + parameters: vec![], + contents: vec![AbstractOp::new(Caller)], + } + .into(), + ]; + + let mut asm = Assembler::new(); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("58335b")); + + Ok(()) + } + + #[test] + fn assemble_instruction_macro_two_delayed_definitions() -> Result<(), Error> { + let ops = vec![ + AbstractOp::new(GetPc), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro0".into(), + parameters: vec![], + }), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro1".into(), + parameters: vec![], + }), + InstructionMacroDefinition { + name: "macro0".into(), + parameters: vec![], + contents: vec![AbstractOp::new(JumpDest)], + } + .into(), + InstructionMacroDefinition { + name: "macro1".into(), + parameters: vec![], + contents: vec![AbstractOp::new(Caller)], + } + .into(), + ]; + + let mut asm = Assembler::new(); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("585b33")); + + Ok(()) + } } From 2a50240a9fe103197ff90b425446e15933393fb9 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 9 Oct 2023 15:39:49 -0300 Subject: [PATCH 31/48] Implementation of the built-ins mentioned in issue 106 --- etk-asm/src/asm.rs | 2 +- etk-asm/src/ast.rs | 1 + etk-asm/src/ingest.rs | 44 +++++++++++++++++++++++++++++++++++++ etk-asm/src/parse/args.rs | 9 ++++++++ etk-asm/src/parse/error.rs | 13 ++++++++++- etk-asm/src/parse/macros.rs | 20 +++++++++++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 3e5556db..a1085ab8 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -157,7 +157,7 @@ pub enum RawOp { /// A new scope to be created with its corresponding list of operations. Scope(Vec), - /// Raw bytes, for example from `%include_hex`, to be included verbatim in + /// Raw bytes, for example from `%include_hex` or %string, to be included verbatim in /// the output. Raw(Vec), } diff --git a/etk-asm/src/ast.rs b/etk-asm/src/ast.rs index e3824c15..e32c93c0 100644 --- a/etk-asm/src/ast.rs +++ b/etk-asm/src/ast.rs @@ -6,6 +6,7 @@ use etk_ops::cancun::Op; #[derive(Debug, Clone, PartialEq)] pub(crate) enum Node { Op(AbstractOp), + Raw(Vec), Import(PathBuf), Include(PathBuf), IncludeHex(PathBuf), diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index bc5e4bfd..389a2047 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -337,6 +337,7 @@ where raws.push(RawOp::Raw(raw)) } + Node::Raw(raw) => raws.push(RawOp::Raw(raw)), } } @@ -373,6 +374,7 @@ mod tests { use std::fmt::Display; use std::io::Write; + use std::path; use super::*; @@ -668,4 +670,46 @@ mod tests { assert_matches!(err, Error::RecursionLimit { .. }); } + + #[test] + fn ingest_hex_builtin() -> Result<(), Error> { + let text = format!( + r#" + jumpdest + %hex("7FAB") + invalid + "# + ); + let (mut _f, root) = new_file(text.clone()); + + let mut output = Vec::new(); + let mut ingest = Ingest::new(&mut output); + ingest.ingest(root, &text)?; + + let expected = hex!("5b7fabfe"); + assert_eq!(output, expected); + + Ok(()) + } + + #[test] + fn ingest_string_builtin() -> Result<(), Error> { + let text = format!( + r#" + jumpdest + %string("hello world") + invalid + "# + ); + let (mut _f, root) = new_file(text.clone()); + + let mut output = Vec::new(); + let mut ingest = Ingest::new(&mut output); + ingest.ingest(root, &text)?; + + let expected = hex!("5b68656c6c6f20776f726c64fe"); + assert_eq!(output, expected); + + Ok(()) + } } diff --git a/etk-asm/src/parse/args.rs b/etk-asm/src/parse/args.rs index f1965378..d590c21f 100644 --- a/etk-asm/src/parse/args.rs +++ b/etk-asm/src/parse/args.rs @@ -21,6 +21,15 @@ impl FromPair for PathBuf { } } +impl FromPair for String { + fn from_pair(pair: Pair) -> Result { + ensure!(pair.as_rule() == Rule::string, error::ArgumentType); + + let txt = pair.as_str(); + Ok(txt[1..txt.len() - 1].to_string()) + } +} + #[derive(Debug, Clone, Eq, PartialEq)] pub(super) struct Label(pub(super) String); diff --git a/etk-asm/src/parse/error.rs b/etk-asm/src/parse/error.rs index a60dcf13..7ebecf2d 100644 --- a/etk-asm/src/parse/error.rs +++ b/etk-asm/src/parse/error.rs @@ -77,7 +77,7 @@ pub enum ParseError { /// An included fail failed to parse as hexadecimal. #[snafu(display("included file `{}` is invalid hex: {}", path.to_string_lossy(), source))] #[non_exhaustive] - InvalidHex { + InvalidHexFile { /// Path to the offending file. path: PathBuf, @@ -87,6 +87,17 @@ pub enum ParseError { /// The location of the error. backtrace: Backtrace, }, + + /// Failed to parse an hexadecimal value. + #[snafu(display("error decoding hexadecimal: {}", value))] + #[non_exhaustive] + InvalidHex { + /// Path to the offending file. + value: String, + + /// The location of the error. + backtrace: Backtrace, + }, } impl From> for ParseError { diff --git a/etk-asm/src/parse/macros.rs b/etk-asm/src/parse/macros.rs index c262917f..1ac77d50 100644 --- a/etk-asm/src/parse/macros.rs +++ b/etk-asm/src/parse/macros.rs @@ -7,6 +7,7 @@ use crate::ops::{ AbstractOp, Expression, ExpressionMacroDefinition, ExpressionMacroInvocation, InstructionMacroDefinition, InstructionMacroInvocation, }; +use crate::parse::error; use pest::iterators::Pair; use std::path::PathBuf; @@ -45,6 +46,25 @@ pub(crate) fn parse_builtin(pair: Pair) -> Result { let expr = expression::parse(pair.into_inner().next().unwrap())?; Node::Op(AbstractOp::Push(expr.into())) } + Rule::inline_hex => { + let args = <(String,)>::parse_arguments(pair.into_inner())?; + let bytes = hex::decode(args.0).map_err(|e| e.to_string()); + + match bytes { + Ok(bytes) => Node::Raw(bytes), + Err(e) => { + return error::InvalidHex { value: e }.fail(); + } + } + } + Rule::inline_string => { + let args = <(String,)>::parse_arguments(pair.into_inner())?; + let raw_string = args.0; + + let bytes = raw_string.as_bytes().to_vec(); + + Node::Raw(bytes) + } _ => unreachable!(), }; From af6123563ae721bb84c155a4f92e4a62a0d60760 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 9 Oct 2023 16:07:00 -0300 Subject: [PATCH 32/48] forgotten file --- etk-asm/src/parse/asm.pest | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etk-asm/src/parse/asm.pest b/etk-asm/src/parse/asm.pest index 19c9f1bd..0238d552 100644 --- a/etk-asm/src/parse/asm.pest +++ b/etk-asm/src/parse/asm.pest @@ -40,12 +40,14 @@ instruction_macro_variable = @{ "$" ~ function_parameter } instruction_macro = !{ "%" ~ function_invocation } local_macro = { !builtin ~ (instruction_macro_definition | instruction_macro | expression_macro_definition) } -builtin = ${ "%" ~ ( import | include | include_hex | push_macro ) } +builtin = ${ "%" ~ ( import | include | include_hex | push_macro | inline_hex | inline_string ) } import = !{ "import" ~ arguments } include = !{ "include" ~ arguments } include_hex = !{ "include_hex" ~ arguments } push_macro = !{ "push" ~ arguments } +inline_hex = !{ "hex" ~ arguments } +inline_string = !{ "string" ~ arguments } arguments = _{ "(" ~ arguments_list? ~ ")" } arguments_list = _{ ( argument ~ "," )* ~ argument? } From 12c609c081bc96812b067b12a45192c1f700e3c9 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 12 Sep 2023 15:41:08 -0300 Subject: [PATCH 33/48] Assembler simplification --- etk-analyze/src/bin/ecfg.rs | 2 +- etk-asm/src/asm.rs | 877 ++++++++---------- etk-asm/src/ast.rs | 5 +- etk-asm/src/ingest.rs | 255 ++--- etk-asm/src/parse/mod.rs | 1 + etk-asm/tests/asm.rs | 27 + etk-asm/tests/asm/every-op/test.etk | 5 + .../undefined-label-undefined-macro.etk | 6 + etk-asm/tests/asm/variable-push/included.etk | 594 ++++++++++++ etk-asm/tests/asm/variable-push/main.etk | 7 + etk-dasm/src/bin/disease.rs | 5 +- 11 files changed, 1107 insertions(+), 677 deletions(-) create mode 100644 etk-asm/tests/asm/every-op/test.etk create mode 100644 etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk create mode 100644 etk-asm/tests/asm/variable-push/included.etk create mode 100644 etk-asm/tests/asm/variable-push/main.etk diff --git a/etk-analyze/src/bin/ecfg.rs b/etk-analyze/src/bin/ecfg.rs index a8fd7a37..7f963c3b 100644 --- a/etk-analyze/src/bin/ecfg.rs +++ b/etk-analyze/src/bin/ecfg.rs @@ -58,7 +58,7 @@ fn run() -> Result<(), Error> { let blocks = separator .take() .into_iter() - .chain(separator.finish().into_iter()) + .chain(separator.finish()) .map(|x| AnnotatedBlock::annotate(&x)); let mut cfg = ControlFlowGraph::new(blocks); diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index a0555961..3e5556db 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -126,45 +126,42 @@ mod error { #[snafu(backtrace)] source: ParseError, }, + + /// An instruction macro was used without being defined. + #[snafu(display("variable `{}` inside macro, was never defined", var))] + #[non_exhaustive] + UndeclaredVariableMacro { + /// The variable that was used without being defined. + var: String, + + /// The location of the error. + backtrace: Backtrace, + }, } } pub use self::error::Error; -use crate::ops::expression::{self, Terminal}; -use crate::ops::{self, AbstractOp, Assemble, Expression, Imm, MacroDefinition}; -use etk_ops::cancun::Op; +use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; +use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; use rand::Rng; -use snafu::OptionExt; -use std::collections::{hash_map, HashMap, HashSet, VecDeque}; +use std::cmp; +use std::collections::{hash_map, HashMap}; -/// An item to be assembled, which can be either an [`AbstractOp`] or a raw byte -/// sequence. +/// An item to be assembled, which can be either an [`AbstractOp`], +/// the inclusion of a new scope or a raw byte sequence. #[derive(Debug, Clone)] pub enum RawOp { /// An instruction to be assembled. Op(AbstractOp), + /// A new scope to be created with its corresponding list of operations. + Scope(Vec), + /// Raw bytes, for example from `%include_hex`, to be included verbatim in /// the output. Raw(Vec), } -impl RawOp { - fn size(&self) -> Option { - match self { - Self::Op(op) => op.size(), - Self::Raw(raw) => Some(raw.len()), - } - } - - fn expr(&self) -> Option<&Expression> { - match self { - Self::Op(op) => op.expr(), - Self::Raw(_) => None, - } - } -} - impl From for RawOp { fn from(op: AbstractOp) -> Self { Self::Op(op) @@ -190,52 +187,42 @@ impl From> for RawOp { /// # /// # use hex_literal::hex; /// let mut asm = Assembler::new(); -/// asm.push_all(vec![ +/// let result = asm.assemble(vec![ /// AbstractOp::new(GetPc), /// ])?; -/// let output = asm.take(); -/// asm.finish()?; -/// # assert_eq!(output, hex!("58")); +/// # assert_eq!(result, hex!("58")); /// # Result::<(), Error>::Ok(()) /// ``` -#[derive(Debug)] +#[derive(Debug, Default)] pub struct Assembler { - /// Assembled ops, ready to be taken. - ready: Vec, + /// Assembled ops. + ready: Vec, - /// Ops that cannot be encoded yet. - pending: VecDeque, - - /// Sum of the size of all the ops in `pending`, or `None` if `pending` contains - /// an unsized op. - pending_len: Option, - - /// Total number of `u8` that have been appended to `ready`. + /// Number of bytes used by the operations in `ready``. concrete_len: usize, - /// Labels, in `pending`, associated with an `AbstractOp::Label`. + /// Labels associated with an `AbstractOp::Label`. declared_labels: HashMap>, - /// Macros, in `pending`, associated with an `AbstractOp::Macro`. + /// Macros associated with an `AbstractOp::Macro`. declared_macros: HashMap, - /// Labels, in `pending`, that have been referred to (ex. with push) but + /// Labels that have been referred to (ex. with push) but /// have not been declared with an `AbstractOp::Label`. - undeclared_labels: HashSet, + undeclared_labels: Vec, } -impl Default for Assembler { - fn default() -> Self { - Self { - ready: Default::default(), - pending: Default::default(), - pending_len: Some(0), - concrete_len: 0, - declared_labels: Default::default(), - declared_macros: Default::default(), - undeclared_labels: Default::default(), - } - } +/// Struct used to keep track of pending label invocations and their positions in code. +#[derive(Debug, Clone)] +struct PendingLabel { + /// The name of the label. + label: String, + + /// Position where the label was invoked. + position: usize, + + /// Whether the label was invoked with a dynamic push or not. + dynamic_push: bool, } impl Assembler { @@ -244,107 +231,127 @@ impl Assembler { Self::default() } - /// Indicate that the input sequence is complete. Returns any errors that - /// may remain. - pub fn finish(self) -> Result<(), Error> { - if let Some(undef) = self.pending.front() { - return match undef { - RawOp::Op(AbstractOp::Macro(invc)) => error::UndeclaredInstructionMacro { - name: invc.name.clone(), - } - .fail(), - RawOp::Op(op) => { - match op - .clone() - .concretize((&self.declared_labels, &self.declared_macros).into()) - { - Ok(_) => unreachable!(), - Err(ops::Error::ContextIncomplete { - source: expression::Error::UnknownMacro { name, .. }, - .. - }) => error::UndeclaredExpressionMacro { name }.fail(), - Err(ops::Error::ContextIncomplete { - source: expression::Error::UnknownLabel { .. }, - .. - }) => { - let labels = op.expr().unwrap().labels(&self.declared_macros).unwrap(); - let declared = self.declared_labels.into_keys().collect(); - let invoked: HashSet<_> = labels.into_iter().collect(); - let missing = invoked - .difference(&declared) - .cloned() - .collect::>(); - error::UndeclaredLabels { labels: missing }.fail() + /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. + fn inspect_macros(&mut self, nodes: &I) -> Result<(), Error> + where + I: IntoIterator + Clone, + O: Into, + { + for op in nodes.clone() { + let op = op.into(); + if let RawOp::Op(AbstractOp::MacroDefinition(_)) = op { + self.declare_macro(op)? + } + } + + Ok(()) + } + + /// Collect any assembled instructions that are ready to be output. + fn take(&mut self) -> Vec { + let output = self.concretize_ops(); + match output { + Ok(v) => { + self.ready.clear(); + v + } + Err(_) => Vec::new(), + } + } + + /// Concretize all assembled instructions. + fn concretize_ops(&mut self) -> Result, Error> { + let mut output = Vec::new(); + for op in self.ready.iter() { + if let RawOp::Op(ref op) = op { + match op + .clone() + .concretize((&self.declared_labels, &self.declared_macros).into()) + { + Ok(cop) => cop.assemble(&mut output), + Err(ops::Error::ContextIncomplete { + source: UnknownLabel { label: _label, .. }, + }) => { + let undeclared_names: Vec<_> = self + .undeclared_labels + .iter() + .map(|PendingLabel { label, .. }| label.clone()) + .collect(); + return error::UndeclaredLabels { + labels: undeclared_names, } - _ => unreachable!(), + .fail(); + } + Err(ops::Error::ContextIncomplete { + source: UnknownMacro { name, .. }, + }) => { + return error::UndeclaredInstructionMacro { name }.fail(); } + Err(ops::Error::ContextIncomplete { + source: UndefinedVariable { name, .. }, + }) => { + return error::UndeclaredVariableMacro { var: name }.fail(); + } + + Err(_) => unreachable!("all ops should be concretizable"), } - // bug: if a variable is used when it isn't available, e.g. push1 $size - _ => unreachable!(), - }; + } else if let RawOp::Raw(raw) = op { + output.extend(raw); + } } - if !self.ready.is_empty() { - panic!("not all assembled bytecode has been taken"); + Ok(output) + } + + /// Check if the input sequence is complete. Returns any errors that + /// may remain. + fn finish(&mut self) -> Result<(), Error> { + if !self.undeclared_labels.is_empty() { + return error::UndeclaredLabels { + labels: self + .undeclared_labels + .iter() + .map(|l| l.label.to_owned()) + .collect::>(), + } + .fail(); } Ok(()) } - /// Collect any assembled instructions that are ready to be output. - pub fn take(&mut self) -> Vec { - std::mem::take(&mut self.ready) - } - /// Feed instructions into the `Assembler`. /// - /// Returns the number of bytes that can be collected with [`Assembler::take`]. - pub fn push_all(&mut self, ops: I) -> Result + /// Returns the code of the assembled program. + pub fn assemble(&mut self, ops: I) -> Result, Error> where - I: IntoIterator, + I: IntoIterator + Clone, O: Into, { + self.inspect_macros(&ops)?; + for op in ops { self.push(op)?; } - Ok(self.ready.len()) + self.finish()?; + + Ok(self.take()) } - /// Insert explicilty declared macros and labels, via `AbstractOp`, and implictly declared - /// macros and labels via usage in `Op`. - fn declare_content(&mut self, rop: &RawOp) -> Result<(), Error> { - match rop { - RawOp::Op(AbstractOp::Label(ref label)) => { - match self.declared_labels.entry(label.to_owned()) { - hash_map::Entry::Occupied(_) => { - return error::DuplicateLabel { label }.fail(); - } - hash_map::Entry::Vacant(v) => { - v.insert(None); - self.undeclared_labels.remove(label); - } - } - } - RawOp::Op(AbstractOp::MacroDefinition(ref defn)) => { - match self.declared_macros.entry(defn.name().to_owned()) { - hash_map::Entry::Occupied(_) => { - return error::DuplicateMacro { name: defn.name() }.fail() - } - hash_map::Entry::Vacant(v) => { - v.insert(defn.to_owned()); - } + /// Insert explicitly declared macros, via `AbstractOp`, into the `Assembler`. + fn declare_macro(&mut self, rop: O) -> Result<(), Error> + where + O: Into, + { + let rop = rop.into(); + if let RawOp::Op(AbstractOp::MacroDefinition(ref defn)) = rop { + match self.declared_macros.entry(defn.name().to_owned()) { + hash_map::Entry::Occupied(_) => { + return error::DuplicateMacro { name: defn.name() }.fail() } - } - _ => (), - }; - - // Get all labels used by `rop`, check if they've been defined, and if not, note them as - // "undeclared". - if let Some(Ok(labels)) = rop.expr().map(|e| e.labels(&self.declared_macros)) { - for label in labels { - if !self.declared_labels.contains_key(&label) { - self.undeclared_labels.insert(label.to_owned()); + hash_map::Entry::Vacant(v) => { + v.insert(defn.to_owned()); } } } @@ -353,39 +360,59 @@ impl Assembler { } /// Feed a single instruction into the `Assembler`. - /// - /// Returns the number of bytes that can be collected with [`Assembler::take`] - pub fn push(&mut self, rop: O) -> Result + fn push(&mut self, rop: O) -> Result where O: Into, { let rop = rop.into(); - self.declare_content(&rop)?; + self.declare_label(&rop)?; - // Expand instruction macros immediately. We do this here because it's the same process - // regardless if we `push_read` or `push_pending` -- in fact, `expand_macro` pushes each op - // individually which calls the correct unchecked push. + // Expand instruction macros immediately. if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { self.expand_macro(&m.name, &m.parameters)?; - return Ok(self.ready.len()); + return Ok(self.concrete_len); } - self.push_unchecked(rop)?; - Ok(self.ready.len()) + self.push_rawop(rop)?; + Ok(self.concrete_len) } - fn push_unchecked(&mut self, rop: RawOp) -> Result<(), Error> { - if self.pending.is_empty() && self.pending_len.is_some() { - self.push_ready(rop) - } else { - self.push_pending(rop) + fn declare_label(&mut self, rop: &RawOp) -> Result<(), Error> { + if let RawOp::Op(AbstractOp::Label(ref label)) = *rop { + match self.declared_labels.entry(label.to_owned()) { + hash_map::Entry::Occupied(_) => { + return error::DuplicateLabel { label }.fail(); + } + hash_map::Entry::Vacant(v) => { + v.insert(None); + } + } } + Ok(()) } - fn push_ready(&mut self, rop: RawOp) -> Result<(), Error> { + fn push_rawop(&mut self, rop: RawOp) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(label)) => { + let mut dst = 0; + for ul in self.undeclared_labels.iter() { + if (ul.label == label) & ul.dynamic_push { + // Compensation in case label was dynamically pushed. + let mut tmp = + ((self.concrete_len as f32 - ul.position as f32) / 256.0).floor(); + + // Size already accounted for %push(label) was 2. + if tmp > 1.0 { + tmp -= 1.0; + } + dst = cmp::max(tmp as usize, dst); + } + } + + self.undeclared_labels.retain(|l| l.label != *label); + self.concrete_len += dst; + let old = self .declared_labels .insert(label, Some(self.concrete_len)) @@ -394,18 +421,7 @@ impl Assembler { Ok(()) } RawOp::Op(AbstractOp::MacroDefinition(_)) => Ok(()), - RawOp::Op(AbstractOp::Macro(ref m)) => { - match self.declared_macros.get(&m.name) { - // Do nothing if the instruction macro has been defined. - Some(MacroDefinition::Instruction(_)) => (), - _ => { - assert_eq!(self.pending_len, Some(0)); - self.pending_len = None; - self.pending.push_back(rop); - } - } - Ok(()) - } + RawOp::Op(AbstractOp::Macro(_)) => Ok(()), RawOp::Op(ref op) => { match op .clone() @@ -413,7 +429,7 @@ impl Assembler { { Ok(cop) => { self.concrete_len += cop.size(); - cop.assemble(&mut self.ready); + self.ready.push(rop) } Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { return error::ExpressionTooLarge { @@ -430,203 +446,48 @@ impl Assembler { } .fail() } - Err(_) => { - assert_eq!(self.pending_len, Some(0)); - self.pending_len = rop.size(); - self.pending.push_back(rop); + Err(ops::Error::ContextIncomplete { + source: UnknownLabel { label: _label, .. }, + }) => { + let mut dynamic_push = false; + match op.size() { + Some(size) => self.concrete_len += size, + None => { + self.concrete_len += 2; + dynamic_push = true; + } + }; + + self.undeclared_labels.push(PendingLabel { + label: _label.to_owned(), + position: self.ready.len(), + dynamic_push, + }); + self.ready.push(rop); } + Err(ops::Error::ContextIncomplete { + source: UnknownMacro { name, .. }, + }) => return error::UndeclaredInstructionMacro { name }.fail(), + Err(ops::Error::ContextIncomplete { + source: UndefinedVariable { name, .. }, + }) => return error::UndeclaredVariableMacro { var: name }.fail(), } Ok(()) } RawOp::Raw(raw) => { self.concrete_len += raw.len(); - self.ready.extend(raw); + self.ready.push(RawOp::Raw(raw)); Ok(()) } - } - } - - fn push_pending(&mut self, rop: RawOp) -> Result<(), Error> { - // Update total size of pending ops. - if let Some(ref mut pending_len) = self.pending_len { - match rop.size() { - Some(size) => *pending_len += size, - None => self.pending_len = None, - } - } - - // Handle new label and macro definitions. - match (self.pending_len, rop) { - (Some(pending_len), RawOp::Op(AbstractOp::Label(lbl))) => { - // The label has a defined address. - let address = self.concrete_len + pending_len; - let item = self.declared_labels.get_mut(&*lbl).unwrap(); - *item = Some(address); - } - (None, rop @ RawOp::Op(AbstractOp::Label(_))) => { - self.pending.push_back(rop); - if self.undeclared_labels.is_empty() { - self.choose_sizes()?; - } - } - (_, RawOp::Op(AbstractOp::MacroDefinition(defn))) => { - if let Some(RawOp::Op(AbstractOp::Macro(invc))) = self.pending.front() { - if defn.name() == &invc.name { - let invc = invc.clone(); - self.pending.pop_front(); - self.expand_macro(&invc.name, &invc.parameters)?; - } - } - } - (_, rop) => { - // Not a label. - self.pending.push_back(rop); - } - } - - // Repeatedly check if the front of the pending list is ready. - while let Some(next) = self.pending.front() { - let op = match next { - RawOp::Op(AbstractOp::Push(Imm { - tree: Expression::Terminal(Terminal::Label(_)), - .. - })) => { - if self.undeclared_labels.is_empty() { - unreachable!() - } else { - // Still waiting on more labels. - break; - } - } - RawOp::Op(AbstractOp::Label(_)) => unreachable!(), - RawOp::Op(AbstractOp::Macro(_)) => { - // Still waiting on more macros. - break; - } - RawOp::Op(op) => op, - RawOp::Raw(_) => { - self.pop_pending()?; - continue; - } - }; - - match op - .clone() - .concretize((&self.declared_labels, &self.declared_macros).into()) - { - Ok(cop) => { - let front = self.pending.front_mut().unwrap(); - *front = RawOp::Op(cop.into()); - } - Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { - return error::ExpressionTooLarge { - expr: op.expr().unwrap().clone(), - value, - spec, - } - .fail(); - } - Err(_) => { - // Still waiting for some definition. - break; - } - } - - self.pop_pending()?; - } - - Ok(()) - } - - fn pop_pending(&mut self) -> Result<(), Error> { - let popped = self.pending.pop_front().unwrap(); - - let size; - - match popped { - RawOp::Raw(raw) => { - size = raw.len(); - self.ready.extend(raw); - } - RawOp::Op(aop) => { - let cop = aop - .concretize((&self.declared_labels, &self.declared_macros).into()) - // Already able to concretize in `push_pending` loop. - .unwrap(); - size = cop.size(); - cop.assemble(&mut self.ready); - } - } - - self.concrete_len += size; - - if self.pending.is_empty() { - self.pending_len = Some(0); - } else if let Some(ref mut pending_len) = self.pending_len { - *pending_len -= size; - } - - Ok(()) - } - - fn choose_sizes(&mut self) -> Result<(), Error> { - let mut sizes: HashMap> = self - .pending - .iter() - .filter(|op| matches!(op, RawOp::Op(AbstractOp::Push(_)))) - .map(|op| (op.expr().unwrap().clone(), Op::<()>::push_for(1).unwrap())) - .collect(); - - let mut subasm; - - loop { - // Create a sub-assembler to try assembling with the sizes in - // `undefined_labels`. - subasm = Self::default(); - subasm.concrete_len = self.concrete_len; - subasm.declared_labels = self.declared_labels.clone(); - - let result: Result, Error> = self - .pending - .iter() - .map(|op| { - let new = match op { - RawOp::Op(AbstractOp::Push(Imm { tree, .. })) => { - let new = sizes[tree].with(tree.clone()).unwrap(); - let aop = AbstractOp::new(new); - RawOp::Op(aop) - } - op => op.clone(), - }; - - subasm.push_pending(new) - }) - .collect(); - - match result { - Ok(_) => { - assert!(subasm.pending.is_empty()); - break; - } - Err(Error::ExpressionTooLarge { expr, .. }) => { - // If an expression is too large for an op, increase the width of that op. - let item = sizes.get_mut(&expr).unwrap(); - let new_size = item.upsize().context(error::UnsizedPushTooLarge)?; - *item = new_size; - } - Err(e) => return Err(e), + RawOp::Scope(scope) => { + let mut asm = Self::new(); + let scope_result = asm.assemble(scope)?; + self.concrete_len += scope_result.len(); + self.ready.push(RawOp::Raw(scope_result)); + Ok(()) } } - - // Insert the results of the sub-assembler into self. - let raw = subasm.take(); - self.pending_len = Some(raw.len()); - self.pending.clear(); - self.pending.push_back(RawOp::Raw(raw)); - self.declared_labels = subasm.declared_labels; - - Ok(()) } fn expand_macro( @@ -686,19 +547,12 @@ impl Assembler { } } - Ok(Some(self.push_all(m.contents)?)) - } - _ => { - assert_eq!(self.pending_len, Some(0)); - self.pending_len = None; - self.pending.push_back(RawOp::Op(AbstractOp::Macro( - ops::InstructionMacroInvocation { - name: name.to_string(), - parameters: parameters.to_vec(), - }, - ))); - Ok(None) + for op in m.contents.iter() { + self.push(op.clone())?; + } + Ok(Some(self.concrete_len)) } + _ => error::UndeclaredInstructionMacro { name }.fail(), } } } @@ -718,20 +572,19 @@ mod tests { #[test] fn assemble_variable_push_const_while_pending() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::Op(Push1(Imm::with_label("label1")).into()), AbstractOp::Push(Terminal::Number(0xaabb.into()).into()), AbstractOp::Label("label1".into()), ])?; - assert_eq!(5, sz); - assert_eq!(asm.take(), hex!("600561aabb")); + assert_eq!(result, hex!("600561aabb")); Ok(()) } #[test] fn assemble_variable_pushes_abab() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("label1")), AbstractOp::Push(Imm::with_label("label2")), @@ -740,15 +593,14 @@ mod tests { AbstractOp::Label("label2".into()), AbstractOp::new(GetPc), ])?; - assert_eq!(7, sz); - assert_eq!(asm.take(), hex!("5b600560065858")); + assert_eq!(result, hex!("5b600560065858")); Ok(()) } #[test] fn assemble_variable_pushes_abba() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("label1")), AbstractOp::Push(Imm::with_label("label2")), @@ -757,33 +609,30 @@ mod tests { AbstractOp::Label("label1".into()), AbstractOp::new(GetPc), ])?; - assert_eq!(7, sz); - assert_eq!(asm.take(), hex!("5b600660055858")); + assert_eq!(result, hex!("5b600660055858")); Ok(()) } #[test] fn assemble_variable_push1_multiple() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), ])?; - assert_eq!(5, sz); - assert_eq!(asm.take(), hex!("5b60056005")); + assert_eq!(result, hex!("5b60056005")); Ok(()) } #[test] fn assemble_variable_push_const() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![AbstractOp::Push( + let result = asm.assemble(vec![AbstractOp::Push( Terminal::Number((0x00aaaaaaaaaaaaaaaaaaaaaaaa as u128).into()).into(), )])?; - assert_eq!(13, sz); - assert_eq!(asm.take(), hex!("6baaaaaaaaaaaaaaaaaaaaaaaa")); + assert_eq!(result, hex!("6baaaaaaaaaaaaaaaaaaaaaaaa")); Ok(()) } @@ -793,7 +642,7 @@ mod tests { let mut asm = Assembler::new(); let err = asm - .push_all(vec![AbstractOp::Push(Terminal::Number(v).into())]) + .assemble(vec![AbstractOp::Push(Terminal::Number(v).into())]) .unwrap_err(); assert_matches!(err, Error::ExpressionTooLarge { .. }); @@ -803,7 +652,7 @@ mod tests { fn assemble_variable_push_negative() { let mut asm = Assembler::new(); let err = asm - .push_all(vec![AbstractOp::Push(Terminal::Number((-1).into()).into())]) + .assemble(vec![AbstractOp::Push(Terminal::Number((-1).into()).into())]) .unwrap_err(); assert_matches!(err, Error::ExpressionNegative { .. }); @@ -812,73 +661,68 @@ mod tests { #[test] fn assemble_variable_push_const0() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![AbstractOp::Push( + let result = asm.assemble(vec![AbstractOp::Push( Terminal::Number((0x00 as u128).into()).into(), )])?; - assert_eq!(2, sz); - assert_eq!(asm.take(), hex!("6000")); + assert_eq!(result, hex!("6000")); Ok(()) } #[test] fn assemble_variable_push1_known() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Label("auto".into()), AbstractOp::Push(Imm::with_label("auto")), ])?; - assert_eq!(3, sz); - assert_eq!(asm.take(), hex!("5b6001")); + assert_eq!(result, hex!("5b6001")); Ok(()) } #[test] fn assemble_variable_push1() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), AbstractOp::new(JumpDest), ])?; - assert_eq!(3, sz); - assert_eq!(asm.take(), hex!("60025b")); + assert_eq!(result, hex!("60025b")); Ok(()) } #[test] fn assemble_variable_push1_reuse() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), AbstractOp::new(JumpDest), AbstractOp::new(Push1(Imm::with_label("auto"))), ])?; - assert_eq!(5, sz); - assert_eq!(asm.take(), hex!("60025b6002")); + assert_eq!(result, hex!("60025b6002")); Ok(()) } #[test] fn assemble_variable_push2() -> Result<(), Error> { - let mut asm = Assembler::new(); - asm.push(AbstractOp::Push(Imm::with_label("auto")))?; + let mut code = vec![]; + code.push(AbstractOp::Push(Imm::with_label("auto"))); for _ in 0..255 { - asm.push(AbstractOp::new(GetPc))?; + code.push(AbstractOp::new(GetPc)); } - asm.push_all(vec![ - AbstractOp::Label("auto".into()), - AbstractOp::new(JumpDest), - ])?; + code.push(AbstractOp::Label("auto".into())); + code.push(AbstractOp::new(JumpDest)); + + let mut asm = Assembler::new(); + let result = asm.assemble(code)?; let mut expected = vec![0x61, 0x01, 0x02]; expected.extend_from_slice(&[0x58; 255]); expected.push(0x5b); - assert_eq!(asm.take(), expected); - - asm.finish()?; + assert_eq!(result, expected); Ok(()) } @@ -886,8 +730,9 @@ mod tests { #[test] fn assemble_undeclared_label() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push_all(vec![AbstractOp::new(Push1(Imm::with_label("hi")))])?; - let err = asm.finish().unwrap_err(); + let err = asm + .assemble(vec![AbstractOp::new(Push1(Imm::with_label("hi")))]) + .unwrap_err(); assert_matches!(err, Error::UndeclaredLabels { labels, .. } if labels == vec!["hi"]); Ok(()) } @@ -895,10 +740,9 @@ mod tests { #[test] fn assemble_jumpdest_no_label() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![AbstractOp::new(JumpDest)])?; - assert_eq!(1, sz); + let result = asm.assemble(vec![AbstractOp::new(JumpDest)])?; assert!(asm.declared_labels.is_empty()); - assert_eq!(asm.take(), hex!("5b")); + assert_eq!(result, hex!("5b")); Ok(()) } @@ -907,11 +751,10 @@ mod tests { let mut asm = Assembler::new(); let ops = vec![AbstractOp::Label("lbl".into()), AbstractOp::new(JumpDest)]; - let sz = asm.push_all(ops)?; - assert_eq!(1, sz); + let result = asm.assemble(ops)?; assert_eq!(asm.declared_labels.len(), 1); assert_eq!(asm.declared_labels.get("lbl"), Some(&Some(0))); - assert_eq!(asm.take(), hex!("5b")); + assert_eq!(result, hex!("5b")); Ok(()) } @@ -924,9 +767,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 3); - assert_eq!(asm.take(), hex!("5b6000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b6000")); Ok(()) } @@ -940,9 +782,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 3); - assert_eq!(asm.take(), hex!("600258")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("600258")); Ok(()) } @@ -956,9 +797,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 3); - assert_eq!(asm.take(), hex!("60025b")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("60025b")); Ok(()) } @@ -972,7 +812,7 @@ mod tests { ops.push(AbstractOp::new(JumpDest)); ops.push(AbstractOp::new(Push1(Imm::with_label("a")))); let mut asm = Assembler::new(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::ExpressionTooLarge { expr: Expression::Terminal(Terminal::Label(label)), .. } if label == "a"); } @@ -985,11 +825,7 @@ mod tests { ops.push(AbstractOp::new(JumpDest)); ops.push(AbstractOp::new(Push1(Imm::with_label("b")))); let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 259); - - let assembled = asm.take(); - asm.finish()?; + let result = asm.assemble(ops)?; let mut expected = vec![0x58; 255]; expected.push(0x5b); @@ -997,7 +833,7 @@ mod tests { expected.push(0x60); expected.push(0xff); - assert_eq!(assembled, expected); + assert_eq!(result, expected); Ok(()) } @@ -1028,10 +864,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 0); - let out = asm.take(); - assert_eq!(out, []); + let result = asm.assemble(ops)?; + assert_eq!(result, []); Ok(()) } @@ -1064,10 +898,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 13); - let out = asm.take(); - assert_eq!(out, hex!("5b60005b600360005b60086000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b60005b600360005b60086000")); Ok(()) } @@ -1096,10 +928,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 8); - let out = asm.take(); - assert_eq!(out, hex!("5b60005b60036000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b60005b60036000")); Ok(()) } @@ -1128,10 +958,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 8); - let out = asm.take(); - assert_eq!(out, hex!("5b60005b60036000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b60005b60036000")); Ok(()) } @@ -1160,9 +988,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(7, sz); - assert_eq!(asm.take(), hex!("5b600560065858")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b600560065858")); Ok(()) } @@ -1173,8 +1000,7 @@ mod tests { InstructionMacroInvocation::with_zero_parameters("my_macro".into()), )]; let mut asm = Assembler::new(); - asm.push_all(ops)?; - let err = asm.finish().unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::UndeclaredInstructionMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1197,7 +1023,7 @@ mod tests { .into(), ]; let mut asm = Assembler::new(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::DuplicateMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1217,7 +1043,7 @@ mod tests { )), ]; let mut asm = Assembler::new(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::DuplicateLabel { label, .. } if label == "a"); Ok(()) @@ -1244,13 +1070,9 @@ mod tests { AbstractOp::new(Push1(Imm::with_label("a"))), ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 5); - - let out = asm.take(); - asm.finish()?; + let result = asm.assemble(ops)?; - assert_eq!(out, hex!("3360016000")); + assert_eq!(result, hex!("3360016000")); Ok(()) } @@ -1280,10 +1102,8 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 7); - let out = asm.take(); - assert_eq!(out, hex!("5b600060426000")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("5b600060426000")); Ok(()) } @@ -1295,10 +1115,8 @@ mod tests { )))]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 2); - let out = asm.take(); - assert_eq!(out, hex!("6002")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("6002")); Ok(()) } @@ -1309,7 +1127,7 @@ mod tests { BigInt::from(-1).into(), )))]; let mut asm = Assembler::new(); - let err = asm.push_all(ops).unwrap_err(); + let err = asm.assemble(ops).unwrap_err(); assert_matches!(err, Error::ExpressionNegative { value, .. } if value == BigInt::from(-1)); Ok(()) @@ -1318,10 +1136,11 @@ mod tests { #[test] fn assemble_expression_undeclared_label() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push_all(vec![AbstractOp::new(Push1(Imm::with_expression( - Terminal::Label(String::from("hi")).into(), - )))])?; - let err = asm.finish().unwrap_err(); + let err = asm + .assemble(vec![AbstractOp::new(Push1(Imm::with_expression( + Terminal::Label(String::from("hi")).into(), + )))]) + .unwrap_err(); assert_matches!(err, Error::UndeclaredLabels { labels, .. } if labels == vec!["hi"]); Ok(()) } @@ -1329,23 +1148,25 @@ mod tests { #[test] fn assemble_variable_push_expression_with_undeclared_labels() -> Result<(), Error> { let mut asm = Assembler::new(); - asm.push_all(vec![ - AbstractOp::new(JumpDest), - AbstractOp::Push(Imm::with_expression(Expression::Plus( - Terminal::Label("foo".into()).into(), - Terminal::Label("bar".into()).into(), - ))), - AbstractOp::new(Gas), - ])?; - let err = asm.finish().unwrap_err(); - assert_matches!(err, Error::UndeclaredLabels { labels, .. } if (labels.contains(&"foo".to_string())) && labels.contains(&"bar".to_string())); + let err = asm + .assemble(vec![ + AbstractOp::new(JumpDest), + AbstractOp::Push(Imm::with_expression(Expression::Plus( + Terminal::Label("foo".into()).into(), + Terminal::Label("bar".into()).into(), + ))), + AbstractOp::new(Gas), + ]) + .unwrap_err(); + // The expressions have short-circuit evaluation, so only the first label is caught in the error. + assert_matches!(err, Error::UndeclaredLabels { labels, .. } if (labels.contains(&"foo".to_string()))); Ok(()) } #[test] fn assemble_variable_push1_expression() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Label("auto".into()), AbstractOp::Push(Imm::with_expression(Expression::Plus( @@ -1353,15 +1174,14 @@ mod tests { Terminal::Label(String::from("auto")).into(), ))), ])?; - assert_eq!(3, sz); - assert_eq!(asm.take(), hex!("5b6002")); + assert_eq!(result, hex!("5b6002")); Ok(()) } #[test] fn assemble_expression_with_labels() -> Result<(), Error> { let mut asm = Assembler::new(); - let sz = asm.push_all(vec![ + let result = asm.assemble(vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_expression(Expression::Plus( Terminal::Label(String::from("foo")).into(), @@ -1371,8 +1191,7 @@ mod tests { AbstractOp::Label("foo".into()), AbstractOp::Label("bar".into()), ])?; - assert_eq!(4, sz); - assert_eq!(asm.take(), hex!("5b60085a")); + assert_eq!(result, hex!("5b60085a")); Ok(()) } @@ -1392,10 +1211,98 @@ mod tests { ]; let mut asm = Assembler::new(); - let sz = asm.push_all(ops)?; - assert_eq!(sz, 2); - let out = asm.take(); - assert_eq!(out, hex!("6002")); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("6002")); + + Ok(()) + } + + #[test] + fn assemble_instruction_macro_with_undeclared_variables() { + let ops = vec![ + InstructionMacroDefinition { + name: "my_macro".into(), + parameters: vec!["foo".into()], + contents: vec![AbstractOp::new(Push1(Imm::with_variable("bar")))], + } + .into(), + AbstractOp::Label("b".into()), + AbstractOp::new(JumpDest), + AbstractOp::new(Push1(Imm::with_label("b"))), + AbstractOp::Macro(InstructionMacroInvocation { + name: "my_macro".into(), + parameters: vec![BigInt::from_bytes_be(Sign::Plus, &vec![0x42]).into()], + }), + ]; + + let mut asm = Assembler::new(); + let err = asm.assemble(ops).unwrap_err(); + + assert_matches!(err, Error::UndeclaredVariableMacro { var, .. } if var == "bar"); + } + + #[test] + fn assemble_instruction_macro_two_delayed_definitions_mirrored() -> Result<(), Error> { + let ops = vec![ + AbstractOp::new(GetPc), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro1".into(), + parameters: vec![], + }), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro0".into(), + parameters: vec![], + }), + InstructionMacroDefinition { + name: "macro0".into(), + parameters: vec![], + contents: vec![AbstractOp::new(JumpDest)], + } + .into(), + InstructionMacroDefinition { + name: "macro1".into(), + parameters: vec![], + contents: vec![AbstractOp::new(Caller)], + } + .into(), + ]; + + let mut asm = Assembler::new(); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("58335b")); + + Ok(()) + } + + #[test] + fn assemble_instruction_macro_two_delayed_definitions() -> Result<(), Error> { + let ops = vec![ + AbstractOp::new(GetPc), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro0".into(), + parameters: vec![], + }), + AbstractOp::Macro(InstructionMacroInvocation { + name: "macro1".into(), + parameters: vec![], + }), + InstructionMacroDefinition { + name: "macro0".into(), + parameters: vec![], + contents: vec![AbstractOp::new(JumpDest)], + } + .into(), + InstructionMacroDefinition { + name: "macro1".into(), + parameters: vec![], + contents: vec![AbstractOp::new(Caller)], + } + .into(), + ]; + + let mut asm = Assembler::new(); + let result = asm.assemble(ops)?; + assert_eq!(result, hex!("585b33")); Ok(()) } diff --git a/etk-asm/src/ast.rs b/etk-asm/src/ast.rs index a4d2e15d..e3824c15 100644 --- a/etk-asm/src/ast.rs +++ b/etk-asm/src/ast.rs @@ -1,16 +1,15 @@ +use std::path::PathBuf; + use crate::ops::{Abstract, AbstractOp, ExpressionMacroDefinition, InstructionMacroDefinition}; use etk_ops::cancun::Op; -use std::path::PathBuf; #[derive(Debug, Clone, PartialEq)] pub(crate) enum Node { Op(AbstractOp), - Raw(Vec), Import(PathBuf), Include(PathBuf), IncludeHex(PathBuf), } - impl From> for Node { fn from(op: Op) -> Self { Node::Op(AbstractOp::Op(op)) diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index 05227609..f6ba6247 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -51,13 +51,15 @@ mod error { }, /// An error that occurred while parsing a file. - #[snafu(context(false))] #[non_exhaustive] - #[snafu(display("parsing failed"))] + #[snafu(display("parsing failed on path `{}`", path.to_string_lossy()))] Parse { /// The underlying source of this error. #[snafu(backtrace)] source: ParseError, + + /// The location of the error. + path: PathBuf, }, /// An error that occurred while assembling a file. @@ -106,40 +108,7 @@ use std::fs::{read_to_string, File}; use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; -fn parse_file>(path: P) -> Result, Error> { - let asm = read_to_string(path.as_ref()).with_context(|_| error::Io { - message: "reading file before parsing", - path: path.as_ref().to_owned(), - })?; - let nodes = parse_asm(&asm)?; - - Ok(nodes) -} - -#[derive(Debug)] -enum Scope { - Same, - Independent(Box), -} - -impl Scope { - fn same() -> Self { - Self::Same - } - - fn independent() -> Self { - Self::Independent(Box::new(Assembler::new())) - } -} - -#[derive(Debug)] -struct Source { - path: PathBuf, - nodes: std::vec::IntoIter, - scope: Scope, -} - -#[derive(Debug)] +#[derive(Debug, Clone)] struct Root { original: PathBuf, canonicalized: PathBuf, @@ -211,135 +180,44 @@ impl Root { } } -#[must_use] -struct PartialSource<'a, W> { - stack: &'a mut SourceStack, - path: PathBuf, - scope: Scope, -} - -impl<'a, W> PartialSource<'a, W> { - fn path(&self) -> &Path { - &self.path - } - - fn push(self, nodes: Vec) -> &'a mut Source { - self.stack.sources.push(Source { - path: self.path, - nodes: nodes.into_iter(), - scope: self.scope, - }); - - self.stack.sources.last_mut().unwrap() - } -} - #[derive(Debug)] -struct SourceStack { - output: W, - sources: Vec, +struct Program { root: Option, + sources: Vec, } -impl SourceStack { - fn new(output: W) -> Self { +impl Program { + fn new(path: PathBuf) -> Self { Self { - output, - sources: Default::default(), - root: Default::default(), + root: Root::new(path.clone()).ok(), + sources: vec![path], } } - fn resolve(&mut self, path: PathBuf, scope: Scope) -> Result, Error> { + fn push_path(&mut self, path: &PathBuf) -> Result { ensure!(self.sources.len() <= 255, error::RecursionLimit); let path = if let Some(ref root) = self.root { let last = self.sources.last().unwrap(); - let dir = match last.path.parent() { + let dir = match last.parent() { Some(s) => s, None => Path::new("./"), }; let candidate = dir.join(path); root.check(&candidate)?; + self.sources.push(candidate.clone()); candidate } else { assert!(self.sources.is_empty()); - self.root = Some(Root::new(path.clone())?); - path + self.root = Some(Root::new(path.to_owned())?); + path.clone() }; - Ok(PartialSource { - stack: self, - path, - scope, - }) + Ok(path) } - fn peek(&mut self) -> Option<&mut Source> { - self.sources.last_mut() - } -} - -impl SourceStack -where - W: Write, -{ - fn pop(&mut self) -> Result<(), Error> { - let popped = self.sources.pop().unwrap(); - - if self.sources.is_empty() { - self.root = None; - } - - let mut asm = match popped.scope { - Scope::Independent(a) => a, - Scope::Same => return Ok(()), - }; - - let raw = asm.take(); - asm.finish()?; - - if raw.is_empty() { - return Ok(()); - } - - if self.sources.is_empty() { - self.output.write_all(&raw).context(error::Io { - message: "writing output", - path: None, - })?; - Ok(()) - } else { - self.write(RawOp::Raw(raw)) - } - } - - fn write(&mut self, mut op: RawOp) -> Result<(), Error> { - if self.sources.is_empty() { - panic!("no sources!"); - } - - for frame in self.sources[1..].iter_mut().rev() { - let asm = match frame.scope { - Scope::Same => continue, - Scope::Independent(ref mut a) => a, - }; - - if 0 == asm.push(op)? { - return Ok(()); - } else { - op = RawOp::Raw(asm.take()); - } - } - - let first_asm = match self.sources[0].scope { - Scope::Independent(ref mut a) => a, - Scope::Same => panic!("sources[0] must be independent"), - }; - - first_asm.push(op)?; - - Ok(()) + fn pop_path(&mut self) { + self.sources.pop(); } } @@ -370,15 +248,13 @@ where /// ``` #[derive(Debug)] pub struct Ingest { - sources: SourceStack, + output: W, } impl Ingest { /// Make a new `Ingest` that writes assembled bytes to `output`. pub fn new(output: W) -> Self { - Self { - sources: SourceStack::new(output), - } + Self { output } } } @@ -403,7 +279,8 @@ where path: path.clone(), })?; - self.ingest(path, &text) + self.ingest(path, &text)?; + Ok(()) } /// Assemble instructions from `src` as if they were read from a file located @@ -412,61 +289,70 @@ where where P: Into, { - let nodes = parse_asm(src)?; - let partial = self.sources.resolve(path.into(), Scope::independent())?; - partial.push(nodes); - - while let Some(source) = self.sources.peek() { - let node = match source.nodes.next() { - Some(n) => n, - None => { - self.sources.pop()?; - continue; - } - }; + let mut program = Program::new(path.into()); + let nodes = self.preprocess(&mut program, src)?; + let mut asm = Assembler::new(); + let raw = asm.assemble(nodes)?; + + self.output.write_all(&raw).context(error::Io { + message: "writing output", + path: None, + })?; + + Ok(()) + } + fn preprocess(&mut self, program: &mut Program, src: &str) -> Result, Error> { + let nodes = parse_asm(src).with_context(|_| error::Parse { + path: program.sources.last().unwrap().clone(), + })?; + let mut raws = Vec::new(); + for node in nodes { match node { Node::Op(op) => { - self.sources.write(RawOp::Op(op))?; - } - Node::Raw(raw) => { - self.sources.write(RawOp::Raw(raw))?; + raws.push(RawOp::Op(op)); } - Node::Import(path) => { - let partial = self.sources.resolve(path, Scope::same())?; - let parsed = parse_file(partial.path())?; - partial.push(parsed); + Node::Import(imp_path) => { + let new_raws = self.resolve_and_ingest(program, imp_path)?; + raws.extend(new_raws); } - Node::Include(path) => { - let partial = self.sources.resolve(path, Scope::independent())?; - let parsed = parse_file(partial.path())?; - partial.push(parsed); + Node::Include(inc_path) => { + let inc_raws = self.resolve_and_ingest(program, inc_path)?; + raws.push(RawOp::Scope(inc_raws)); } - Node::IncludeHex(path) => { - let partial = self.sources.resolve(path, Scope::same())?; - - let file = - std::fs::read_to_string(partial.path()).with_context(|_| error::Io { - message: "reading hex include", - path: partial.path().to_owned(), - })?; + Node::IncludeHex(hex_path) => { + let file = std::fs::read_to_string(&hex_path).with_context(|_| error::Io { + message: "reading hex include", + path: hex_path.to_owned(), + })?; let raw = hex::decode(file.trim()) .map_err(|e| Box::new(e) as Box) .context(error::InvalidHex { - path: partial.path().to_owned(), + path: hex_path.to_owned(), })?; - partial.push(vec![Node::Raw(raw)]); + raws.push(RawOp::Raw(raw)) } } } - if !self.sources.sources.is_empty() { - panic!("extra sources?"); - } + Ok(raws) + } - Ok(()) + fn resolve_and_ingest( + &mut self, + program: &mut Program, + path: PathBuf, + ) -> Result, Error> { + let source = program.push_path(&path)?; + let code = read_to_string(source).with_context(|_| error::Io { + message: "reading file before parsing", + path: path.to_owned(), + })?; + let new_raws = self.preprocess(program, &code)?; + program.pop_path(); + Ok(new_raws) } } @@ -538,6 +424,7 @@ mod tests { let mut output = Vec::new(); let mut ingest = Ingest::new(&mut output); ingest.ingest(root, &text)?; + assert_eq!(output, hex!("60015b586000566002")); Ok(()) diff --git a/etk-asm/src/parse/mod.rs b/etk-asm/src/parse/mod.rs index dba643c7..4858abaa 100644 --- a/etk-asm/src/parse/mod.rs +++ b/etk-asm/src/parse/mod.rs @@ -19,6 +19,7 @@ use self::{ error::ParseError, parser::{AsmParser, Rule}, }; + use crate::ast::Node; use crate::ops::AbstractOp; use etk_ops::cancun::Op; diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 36fc72cf..c09dba9f 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -108,6 +108,22 @@ fn instruction_macro_with_two_instructions_per_line() { assert_matches!(err, Error::Parse { .. }); } +#[test] +fn undefined_label_undefined_macro() { + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + let err = ingester + .ingest_file(source(&[ + "instruction-macro", + "undefined-label-undefined-macro.etk", + ])) + .unwrap_err(); + + assert_matches!(err, etk_asm::ingest::Error::Assemble { source: + etk_asm::asm::Error::UndeclaredInstructionMacro { name, .. }, .. + } if name == "revert".to_string()); +} + #[test] fn every_op() -> Result<(), Error> { let mut output = Vec::new(); @@ -276,3 +292,14 @@ fn every_op() -> Result<(), Error> { Ok(()) } + +#[test] +fn test_dynamic_push_and_include() -> Result<(), Error> { + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + ingester.ingest_file(source(&["variable-push", "main.etk"]))?; + + assert_eq!(output, hex!("61025758585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585801010101010158585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858015b58")); + + Ok(()) +} diff --git a/etk-asm/tests/asm/every-op/test.etk b/etk-asm/tests/asm/every-op/test.etk new file mode 100644 index 00000000..e1df3285 --- /dev/null +++ b/etk-asm/tests/asm/every-op/test.etk @@ -0,0 +1,5 @@ +%push(hello) +jump + +hello: +jumpdest \ No newline at end of file diff --git a/etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk b/etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk new file mode 100644 index 00000000..4103fb6b --- /dev/null +++ b/etk-asm/tests/asm/instruction-macro/undefined-label-undefined-macro.etk @@ -0,0 +1,6 @@ +%macro revert_if_neq() + push1 revert +%end + +%revert_if_neq() +%revert() \ No newline at end of file diff --git a/etk-asm/tests/asm/variable-push/included.etk b/etk-asm/tests/asm/variable-push/included.etk new file mode 100644 index 00000000..c55517ab --- /dev/null +++ b/etk-asm/tests/asm/variable-push/included.etk @@ -0,0 +1,594 @@ +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +add +add +add +add +add +add +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +pc +add diff --git a/etk-asm/tests/asm/variable-push/main.etk b/etk-asm/tests/asm/variable-push/main.etk new file mode 100644 index 00000000..cef9faf7 --- /dev/null +++ b/etk-asm/tests/asm/variable-push/main.etk @@ -0,0 +1,7 @@ +%push(label) +pc +pc +%include("included.etk") +label: +jumpdest +pc \ No newline at end of file diff --git a/etk-dasm/src/bin/disease.rs b/etk-dasm/src/bin/disease.rs index 7729d9e0..08afb442 100644 --- a/etk-dasm/src/bin/disease.rs +++ b/etk-dasm/src/bin/disease.rs @@ -55,10 +55,7 @@ fn run() -> Result<(), Error> { separator.push_all(disasm.ops()); - let basic_blocks = separator - .take() - .into_iter() - .chain(separator.finish().into_iter()); + let basic_blocks = separator.take().into_iter().chain(separator.finish()); for block in basic_blocks { let mut offset = block.offset; From eb366966b2ec17acdb4ba0402e6f638c61053f5a Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 30 Oct 2023 16:39:10 -0300 Subject: [PATCH 34/48] addressing comments --- etk-asm/src/asm.rs | 255 ++++++++++++++++++++---------------------- etk-asm/src/ingest.rs | 2 +- 2 files changed, 122 insertions(+), 135 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 3e5556db..07271003 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -174,6 +174,12 @@ impl From> for RawOp { } } +impl> From<&O> for RawOp { + fn from(item: &O) -> Self { + item.clone().into() + } +} + /// Assembles a series of [`RawOp`] into raw bytes, tracking and resolving macros and labels, /// and handling dynamic pushes. /// @@ -187,9 +193,8 @@ impl From> for RawOp { /// # /// # use hex_literal::hex; /// let mut asm = Assembler::new(); -/// let result = asm.assemble(vec![ -/// AbstractOp::new(GetPc), -/// ])?; +/// let code = ec![AbstractOp::new(GetPc)] +/// let result = asm.assemble(&code)?; /// # assert_eq!(result, hex!("58")); /// # Result::<(), Error>::Ok(()) /// ``` @@ -232,12 +237,11 @@ impl Assembler { } /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. - fn inspect_macros(&mut self, nodes: &I) -> Result<(), Error> + fn inspect_macros(&mut self, nodes: &[O]) -> Result<(), Error> where - I: IntoIterator + Clone, O: Into, { - for op in nodes.clone() { + for op in nodes { let op = op.into(); if let RawOp::Op(AbstractOp::MacroDefinition(_)) = op { self.declare_macro(op)? @@ -247,18 +251,6 @@ impl Assembler { Ok(()) } - /// Collect any assembled instructions that are ready to be output. - fn take(&mut self) -> Vec { - let output = self.concretize_ops(); - match output { - Ok(v) => { - self.ready.clear(); - v - } - Err(_) => Vec::new(), - } - } - /// Concretize all assembled instructions. fn concretize_ops(&mut self) -> Result, Error> { let mut output = Vec::new(); @@ -270,7 +262,7 @@ impl Assembler { { Ok(cop) => cop.assemble(&mut output), Err(ops::Error::ContextIncomplete { - source: UnknownLabel { label: _label, .. }, + source: UnknownLabel { .. }, }) => { let undeclared_names: Vec<_> = self .undeclared_labels @@ -323,9 +315,8 @@ impl Assembler { /// Feed instructions into the `Assembler`. /// /// Returns the code of the assembled program. - pub fn assemble(&mut self, ops: I) -> Result, Error> + pub fn assemble(&mut self, ops: &[O]) -> Result, Error> where - I: IntoIterator + Clone, O: Into, { self.inspect_macros(&ops)?; @@ -336,7 +327,9 @@ impl Assembler { self.finish()?; - Ok(self.take()) + let output = self.concretize_ops()?; + self.ready.clear(); + Ok(output) } /// Insert explicitly declared macros, via `AbstractOp`, into the `Assembler`. @@ -360,7 +353,7 @@ impl Assembler { } /// Feed a single instruction into the `Assembler`. - fn push(&mut self, rop: O) -> Result + fn push(&mut self, rop: &O) -> Result where O: Into, { @@ -368,31 +361,6 @@ impl Assembler { self.declare_label(&rop)?; - // Expand instruction macros immediately. - if let RawOp::Op(AbstractOp::Macro(ref m)) = rop { - self.expand_macro(&m.name, &m.parameters)?; - return Ok(self.concrete_len); - } - - self.push_rawop(rop)?; - Ok(self.concrete_len) - } - - fn declare_label(&mut self, rop: &RawOp) -> Result<(), Error> { - if let RawOp::Op(AbstractOp::Label(ref label)) = *rop { - match self.declared_labels.entry(label.to_owned()) { - hash_map::Entry::Occupied(_) => { - return error::DuplicateLabel { label }.fail(); - } - hash_map::Entry::Vacant(v) => { - v.insert(None); - } - } - } - Ok(()) - } - - fn push_rawop(&mut self, rop: RawOp) -> Result<(), Error> { match rop { RawOp::Op(AbstractOp::Label(label)) => { let mut dst = 0; @@ -418,10 +386,11 @@ impl Assembler { .insert(label, Some(self.concrete_len)) .expect("label should exist"); assert_eq!(old, None, "label should have been undefined"); - Ok(()) } - RawOp::Op(AbstractOp::MacroDefinition(_)) => Ok(()), - RawOp::Op(AbstractOp::Macro(_)) => Ok(()), + RawOp::Op(AbstractOp::MacroDefinition(_)) => {} + RawOp::Op(AbstractOp::Macro(ref m)) => { + self.expand_macro(&m.name, &m.parameters)?; + } RawOp::Op(ref op) => { match op .clone() @@ -450,13 +419,12 @@ impl Assembler { source: UnknownLabel { label: _label, .. }, }) => { let mut dynamic_push = false; - match op.size() { - Some(size) => self.concrete_len += size, - None => { - self.concrete_len += 2; - dynamic_push = true; - } - }; + if let AbstractOp::Push(_) = op { + dynamic_push = true; + self.concrete_len += 2; + } else { + self.concrete_len += op.size().unwrap(); + } self.undeclared_labels.push(PendingLabel { label: _label.to_owned(), @@ -472,22 +440,34 @@ impl Assembler { source: UndefinedVariable { name, .. }, }) => return error::UndeclaredVariableMacro { var: name }.fail(), } - - Ok(()) } RawOp::Raw(raw) => { self.concrete_len += raw.len(); self.ready.push(RawOp::Raw(raw)); - Ok(()) } RawOp::Scope(scope) => { let mut asm = Self::new(); - let scope_result = asm.assemble(scope)?; + let scope_result = asm.assemble(&scope)?; self.concrete_len += scope_result.len(); self.ready.push(RawOp::Raw(scope_result)); - Ok(()) } } + + Ok(self.concrete_len) + } + + fn declare_label(&mut self, rop: &RawOp) -> Result<(), Error> { + if let RawOp::Op(AbstractOp::Label(ref label)) = *rop { + match self.declared_labels.entry(label.to_owned()) { + hash_map::Entry::Occupied(_) => { + return error::DuplicateLabel { label }.fail(); + } + hash_map::Entry::Vacant(v) => { + v.insert(None); + } + } + } + Ok(()) } fn expand_macro( @@ -548,7 +528,7 @@ impl Assembler { } for op in m.contents.iter() { - self.push(op.clone())?; + self.push(op)?; } Ok(Some(self.concrete_len)) } @@ -572,11 +552,12 @@ mod tests { #[test] fn assemble_variable_push_const_while_pending() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let code = vec![ AbstractOp::Op(Push1(Imm::with_label("label1")).into()), AbstractOp::Push(Terminal::Number(0xaabb.into()).into()), AbstractOp::Label("label1".into()), - ])?; + ]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("600561aabb")); Ok(()) } @@ -584,7 +565,7 @@ mod tests { #[test] fn assemble_variable_pushes_abab() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let code = vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("label1")), AbstractOp::Push(Imm::with_label("label2")), @@ -592,7 +573,8 @@ mod tests { AbstractOp::new(GetPc), AbstractOp::Label("label2".into()), AbstractOp::new(GetPc), - ])?; + ]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("5b600560065858")); Ok(()) } @@ -600,7 +582,7 @@ mod tests { #[test] fn assemble_variable_pushes_abba() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let code = vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("label1")), AbstractOp::Push(Imm::with_label("label2")), @@ -608,7 +590,8 @@ mod tests { AbstractOp::new(GetPc), AbstractOp::Label("label1".into()), AbstractOp::new(GetPc), - ])?; + ]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("5b600660055858")); Ok(()) } @@ -616,12 +599,13 @@ mod tests { #[test] fn assemble_variable_push1_multiple() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let code = vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), - ])?; + ]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("5b60056005")); Ok(()) } @@ -629,9 +613,10 @@ mod tests { #[test] fn assemble_variable_push_const() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![AbstractOp::Push( + let code = vec![AbstractOp::Push( Terminal::Number((0x00aaaaaaaaaaaaaaaaaaaaaaaa as u128).into()).into(), - )])?; + )]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("6baaaaaaaaaaaaaaaaaaaaaaaa")); Ok(()) } @@ -641,9 +626,8 @@ mod tests { let v = BigInt::from_bytes_be(Sign::Plus, &[1u8; 33]); let mut asm = Assembler::new(); - let err = asm - .assemble(vec![AbstractOp::Push(Terminal::Number(v).into())]) - .unwrap_err(); + let code = vec![AbstractOp::Push(Terminal::Number(v).into())]; + let err = asm.assemble(&code).unwrap_err(); assert_matches!(err, Error::ExpressionTooLarge { .. }); } @@ -651,9 +635,8 @@ mod tests { #[test] fn assemble_variable_push_negative() { let mut asm = Assembler::new(); - let err = asm - .assemble(vec![AbstractOp::Push(Terminal::Number((-1).into()).into())]) - .unwrap_err(); + let code = vec![AbstractOp::Push(Terminal::Number((-1).into()).into())]; + let err = asm.assemble(&code).unwrap_err(); assert_matches!(err, Error::ExpressionNegative { .. }); } @@ -661,9 +644,10 @@ mod tests { #[test] fn assemble_variable_push_const0() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![AbstractOp::Push( + let code = vec![AbstractOp::Push( Terminal::Number((0x00 as u128).into()).into(), - )])?; + )]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("6000")); Ok(()) } @@ -671,11 +655,12 @@ mod tests { #[test] fn assemble_variable_push1_known() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let code = vec![ AbstractOp::new(JumpDest), AbstractOp::Label("auto".into()), AbstractOp::Push(Imm::with_label("auto")), - ])?; + ]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("5b6001")); Ok(()) } @@ -683,11 +668,12 @@ mod tests { #[test] fn assemble_variable_push1() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let code = vec![ AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), AbstractOp::new(JumpDest), - ])?; + ]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("60025b")); Ok(()) } @@ -695,12 +681,13 @@ mod tests { #[test] fn assemble_variable_push1_reuse() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let code = vec![ AbstractOp::Push(Imm::with_label("auto")), AbstractOp::Label("auto".into()), AbstractOp::new(JumpDest), AbstractOp::new(Push1(Imm::with_label("auto"))), - ])?; + ]; + let result = asm.assemble(&code)?; assert_eq!(result, hex!("60025b6002")); Ok(()) } @@ -717,7 +704,7 @@ mod tests { code.push(AbstractOp::new(JumpDest)); let mut asm = Assembler::new(); - let result = asm.assemble(code)?; + let result = asm.assemble(&code)?; let mut expected = vec![0x61, 0x01, 0x02]; expected.extend_from_slice(&[0x58; 255]); @@ -730,9 +717,8 @@ mod tests { #[test] fn assemble_undeclared_label() -> Result<(), Error> { let mut asm = Assembler::new(); - let err = asm - .assemble(vec![AbstractOp::new(Push1(Imm::with_label("hi")))]) - .unwrap_err(); + let code = vec![AbstractOp::new(Push1(Imm::with_label("hi")))]; + let err = asm.assemble(&code).unwrap_err(); assert_matches!(err, Error::UndeclaredLabels { labels, .. } if labels == vec!["hi"]); Ok(()) } @@ -740,7 +726,8 @@ mod tests { #[test] fn assemble_jumpdest_no_label() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![AbstractOp::new(JumpDest)])?; + let code = vec![AbstractOp::new(JumpDest)]; + let result = asm.assemble(&code)?; assert!(asm.declared_labels.is_empty()); assert_eq!(result, hex!("5b")); Ok(()) @@ -751,7 +738,7 @@ mod tests { let mut asm = Assembler::new(); let ops = vec![AbstractOp::Label("lbl".into()), AbstractOp::new(JumpDest)]; - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(asm.declared_labels.len(), 1); assert_eq!(asm.declared_labels.get("lbl"), Some(&Some(0))); assert_eq!(result, hex!("5b")); @@ -767,7 +754,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b6000")); Ok(()) @@ -782,7 +769,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("600258")); Ok(()) @@ -797,7 +784,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("60025b")); Ok(()) @@ -812,7 +799,7 @@ mod tests { ops.push(AbstractOp::new(JumpDest)); ops.push(AbstractOp::new(Push1(Imm::with_label("a")))); let mut asm = Assembler::new(); - let err = asm.assemble(ops).unwrap_err(); + let err = asm.assemble(&ops).unwrap_err(); assert_matches!(err, Error::ExpressionTooLarge { expr: Expression::Terminal(Terminal::Label(label)), .. } if label == "a"); } @@ -825,7 +812,7 @@ mod tests { ops.push(AbstractOp::new(JumpDest)); ops.push(AbstractOp::new(Push1(Imm::with_label("b")))); let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; let mut expected = vec![0x58; 255]; expected.push(0x5b); @@ -864,7 +851,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, []); Ok(()) @@ -898,7 +885,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b60005b600360005b60086000")); Ok(()) @@ -928,7 +915,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b60005b60036000")); Ok(()) @@ -958,7 +945,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b60005b60036000")); Ok(()) @@ -988,7 +975,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b600560065858")); Ok(()) @@ -1000,7 +987,7 @@ mod tests { InstructionMacroInvocation::with_zero_parameters("my_macro".into()), )]; let mut asm = Assembler::new(); - let err = asm.assemble(ops).unwrap_err(); + let err = asm.assemble(&ops).unwrap_err(); assert_matches!(err, Error::UndeclaredInstructionMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1023,7 +1010,7 @@ mod tests { .into(), ]; let mut asm = Assembler::new(); - let err = asm.assemble(ops).unwrap_err(); + let err = asm.assemble(&ops).unwrap_err(); assert_matches!(err, Error::DuplicateMacro { name, .. } if name == "my_macro"); Ok(()) @@ -1043,7 +1030,7 @@ mod tests { )), ]; let mut asm = Assembler::new(); - let err = asm.assemble(ops).unwrap_err(); + let err = asm.assemble(&ops).unwrap_err(); assert_matches!(err, Error::DuplicateLabel { label, .. } if label == "a"); Ok(()) @@ -1070,7 +1057,7 @@ mod tests { AbstractOp::new(Push1(Imm::with_label("a"))), ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("3360016000")); @@ -1102,7 +1089,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b600060426000")); Ok(()) @@ -1115,7 +1102,7 @@ mod tests { )))]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("6002")); Ok(()) @@ -1127,7 +1114,7 @@ mod tests { BigInt::from(-1).into(), )))]; let mut asm = Assembler::new(); - let err = asm.assemble(ops).unwrap_err(); + let err = asm.assemble(&ops).unwrap_err(); assert_matches!(err, Error::ExpressionNegative { value, .. } if value == BigInt::from(-1)); Ok(()) @@ -1136,11 +1123,10 @@ mod tests { #[test] fn assemble_expression_undeclared_label() -> Result<(), Error> { let mut asm = Assembler::new(); - let err = asm - .assemble(vec![AbstractOp::new(Push1(Imm::with_expression( - Terminal::Label(String::from("hi")).into(), - )))]) - .unwrap_err(); + let ops = vec![AbstractOp::new(Push1(Imm::with_expression( + Terminal::Label(String::from("hi")).into(), + )))]; + let err = asm.assemble(&ops).unwrap_err(); assert_matches!(err, Error::UndeclaredLabels { labels, .. } if labels == vec!["hi"]); Ok(()) } @@ -1148,16 +1134,15 @@ mod tests { #[test] fn assemble_variable_push_expression_with_undeclared_labels() -> Result<(), Error> { let mut asm = Assembler::new(); - let err = asm - .assemble(vec![ - AbstractOp::new(JumpDest), - AbstractOp::Push(Imm::with_expression(Expression::Plus( - Terminal::Label("foo".into()).into(), - Terminal::Label("bar".into()).into(), - ))), - AbstractOp::new(Gas), - ]) - .unwrap_err(); + let ops = vec![ + AbstractOp::new(JumpDest), + AbstractOp::Push(Imm::with_expression(Expression::Plus( + Terminal::Label("foo".into()).into(), + Terminal::Label("bar".into()).into(), + ))), + AbstractOp::new(Gas), + ]; + let err = asm.assemble(&ops).unwrap_err(); // The expressions have short-circuit evaluation, so only the first label is caught in the error. assert_matches!(err, Error::UndeclaredLabels { labels, .. } if (labels.contains(&"foo".to_string()))); Ok(()) @@ -1166,14 +1151,15 @@ mod tests { #[test] fn assemble_variable_push1_expression() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let ops = vec![ AbstractOp::new(JumpDest), AbstractOp::Label("auto".into()), AbstractOp::Push(Imm::with_expression(Expression::Plus( 1.into(), Terminal::Label(String::from("auto")).into(), ))), - ])?; + ]; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b6002")); Ok(()) } @@ -1181,7 +1167,7 @@ mod tests { #[test] fn assemble_expression_with_labels() -> Result<(), Error> { let mut asm = Assembler::new(); - let result = asm.assemble(vec![ + let ops = vec![ AbstractOp::new(JumpDest), AbstractOp::Push(Imm::with_expression(Expression::Plus( Terminal::Label(String::from("foo")).into(), @@ -1190,7 +1176,8 @@ mod tests { AbstractOp::new(Gas), AbstractOp::Label("foo".into()), AbstractOp::Label("bar".into()), - ])?; + ]; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("5b60085a")); Ok(()) } @@ -1211,7 +1198,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("6002")); Ok(()) @@ -1236,7 +1223,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let err = asm.assemble(ops).unwrap_err(); + let err = asm.assemble(&ops).unwrap_err(); assert_matches!(err, Error::UndeclaredVariableMacro { var, .. } if var == "bar"); } @@ -1268,7 +1255,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("58335b")); Ok(()) @@ -1301,7 +1288,7 @@ mod tests { ]; let mut asm = Assembler::new(); - let result = asm.assemble(ops)?; + let result = asm.assemble(&ops)?; assert_eq!(result, hex!("585b33")); Ok(()) diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index f6ba6247..0cef990d 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -292,7 +292,7 @@ where let mut program = Program::new(path.into()); let nodes = self.preprocess(&mut program, src)?; let mut asm = Assembler::new(); - let raw = asm.assemble(nodes)?; + let raw = asm.assemble(&nodes)?; self.output.write_all(&raw).context(error::Io { message: "writing output", From c355aa2d0eaa8059de1ed4b8bb4b00ca6538b76e Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Mon, 30 Oct 2023 16:55:42 -0300 Subject: [PATCH 35/48] addressing comments --- etk-asm/src/asm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 07271003..7c7121d9 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -176,7 +176,7 @@ impl From> for RawOp { impl> From<&O> for RawOp { fn from(item: &O) -> Self { - item.clone().into() + item.into() } } @@ -319,7 +319,7 @@ impl Assembler { where O: Into, { - self.inspect_macros(&ops)?; + self.inspect_macros(ops)?; for op in ops { self.push(op)?; From 27fc550804f7c07a75fa2eb9bbb1dac73c82de08 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 31 Oct 2023 11:18:43 -0300 Subject: [PATCH 36/48] Dynamic push: new test --- etk-asm/src/asm.rs | 44 +++++++++++------------ etk-asm/tests/asm.rs | 13 +++++++ etk-asm/tests/asm/variable-push2/main.etk | 3 ++ 3 files changed, 38 insertions(+), 22 deletions(-) create mode 100644 etk-asm/tests/asm/variable-push2/main.etk diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 7c7121d9..1786d89d 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -142,7 +142,7 @@ mod error { pub use self::error::Error; use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; -use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; +use crate::ops::{self, AbstractOp, Assemble, Expression, Imm, MacroDefinition}; use rand::Rng; use std::cmp; use std::collections::{hash_map, HashMap}; @@ -174,12 +174,6 @@ impl From> for RawOp { } } -impl> From<&O> for RawOp { - fn from(item: &O) -> Self { - item.into() - } -} - /// Assembles a series of [`RawOp`] into raw bytes, tracking and resolving macros and labels, /// and handling dynamic pushes. /// @@ -226,6 +220,9 @@ struct PendingLabel { /// Position where the label was invoked. position: usize, + /// The immediate value of the label invocation. + imm: Option, + /// Whether the label was invoked with a dynamic push or not. dynamic_push: bool, } @@ -239,12 +236,12 @@ impl Assembler { /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. fn inspect_macros(&mut self, nodes: &[O]) -> Result<(), Error> where - O: Into, + O: Into + Clone, //TODO: Remove clone and fix code { - for op in nodes { - let op = op.into(); - if let RawOp::Op(AbstractOp::MacroDefinition(_)) = op { - self.declare_macro(op)? + for op in nodes.iter() { + let raw_op: RawOp = (*op).clone().into(); + if let RawOp::Op(AbstractOp::MacroDefinition(_)) = raw_op { + self.declare_macro(raw_op)? } } @@ -317,7 +314,7 @@ impl Assembler { /// Returns the code of the assembled program. pub fn assemble(&mut self, ops: &[O]) -> Result, Error> where - O: Into, + O: Into + Clone, //TODO: Remove clone and fix code { self.inspect_macros(ops)?; @@ -355,9 +352,10 @@ impl Assembler { /// Feed a single instruction into the `Assembler`. fn push(&mut self, rop: &O) -> Result where - O: Into, + O: Into + Clone, { - let rop = rop.into(); + //let rop = rop.into(); + let rop = rop.clone().into(); self.declare_label(&rop)?; @@ -367,17 +365,16 @@ impl Assembler { for ul in self.undeclared_labels.iter() { if (ul.label == label) & ul.dynamic_push { // Compensation in case label was dynamically pushed. - let mut tmp = - ((self.concrete_len as f32 - ul.position as f32) / 256.0).floor(); + let mut dst_tmp = (self.concrete_len - ul.position) / 256; // Size already accounted for %push(label) was 2. - if tmp > 1.0 { - tmp -= 1.0; + if dst_tmp > 1 { + dst_tmp -= 1; } - dst = cmp::max(tmp as usize, dst); + + dst = cmp::max(dst_tmp, dst); } } - self.undeclared_labels.retain(|l| l.label != *label); self.concrete_len += dst; @@ -419,9 +416,11 @@ impl Assembler { source: UnknownLabel { label: _label, .. }, }) => { let mut dynamic_push = false; - if let AbstractOp::Push(_) = op { + let mut imm: Option = None; + if let AbstractOp::Push(inner) = op { dynamic_push = true; self.concrete_len += 2; + imm = Some(inner.clone()); } else { self.concrete_len += op.size().unwrap(); } @@ -429,6 +428,7 @@ impl Assembler { self.undeclared_labels.push(PendingLabel { label: _label.to_owned(), position: self.ready.len(), + imm, dynamic_push, }); self.ready.push(rop); diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index c09dba9f..88ae6b9b 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -303,3 +303,16 @@ fn test_dynamic_push_and_include() -> Result<(), Error> { Ok(()) } + +#[test] +fn test_dynamic_push2() -> Result<(), Error> { + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + ingester.ingest_file(source(&["variable-push2", "main.etk"]))?; + + println!("{:x?}", output); + println!("{:?}", output); + assert_eq!(output, hex!("61010058")); + + Ok(()) +} diff --git a/etk-asm/tests/asm/variable-push2/main.etk b/etk-asm/tests/asm/variable-push2/main.etk new file mode 100644 index 00000000..618285ee --- /dev/null +++ b/etk-asm/tests/asm/variable-push2/main.etk @@ -0,0 +1,3 @@ +%push(label + 254) +label: +pc \ No newline at end of file From a549703cccb730b65a35a940917a6dffcfb0f146 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Tue, 31 Oct 2023 11:20:02 -0300 Subject: [PATCH 37/48] Dynamic push: new test --- etk-asm/src/asm.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 1786d89d..a6158e31 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -220,9 +220,6 @@ struct PendingLabel { /// Position where the label was invoked. position: usize, - /// The immediate value of the label invocation. - imm: Option, - /// Whether the label was invoked with a dynamic push or not. dynamic_push: bool, } @@ -416,11 +413,9 @@ impl Assembler { source: UnknownLabel { label: _label, .. }, }) => { let mut dynamic_push = false; - let mut imm: Option = None; - if let AbstractOp::Push(inner) = op { + if let AbstractOp::Push(_) = op { dynamic_push = true; self.concrete_len += 2; - imm = Some(inner.clone()); } else { self.concrete_len += op.size().unwrap(); } @@ -428,7 +423,6 @@ impl Assembler { self.undeclared_labels.push(PendingLabel { label: _label.to_owned(), position: self.ready.len(), - imm, dynamic_push, }); self.ready.push(rop); From 91a797e0acc62022ae916e55d6c7cfaf02dcb41b Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Wed, 8 Nov 2023 12:46:25 -0300 Subject: [PATCH 38/48] Minor fix and new test --- etk-asm/src/asm.rs | 10 +++++----- etk-asm/tests/asm.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index a6158e31..83dccc20 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -142,7 +142,7 @@ mod error { pub use self::error::Error; use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; -use crate::ops::{self, AbstractOp, Assemble, Expression, Imm, MacroDefinition}; +use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; use rand::Rng; use std::cmp; use std::collections::{hash_map, HashMap}; @@ -187,7 +187,7 @@ impl From> for RawOp { /// # /// # use hex_literal::hex; /// let mut asm = Assembler::new(); -/// let code = ec![AbstractOp::new(GetPc)] +/// let code = vec![AbstractOp::new(GetPc)]; /// let result = asm.assemble(&code)?; /// # assert_eq!(result, hex!("58")); /// # Result::<(), Error>::Ok(()) @@ -349,7 +349,7 @@ impl Assembler { /// Feed a single instruction into the `Assembler`. fn push(&mut self, rop: &O) -> Result where - O: Into + Clone, + O: Into + Clone, //TODO: Remove clone and fix code { //let rop = rop.into(); let rop = rop.clone().into(); @@ -410,7 +410,7 @@ impl Assembler { .fail() } Err(ops::Error::ContextIncomplete { - source: UnknownLabel { label: _label, .. }, + source: UnknownLabel { label, .. }, }) => { let mut dynamic_push = false; if let AbstractOp::Push(_) = op { @@ -421,7 +421,7 @@ impl Assembler { } self.undeclared_labels.push(PendingLabel { - label: _label.to_owned(), + label, position: self.ready.len(), dynamic_push, }); diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index 88ae6b9b..c7d659c1 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -312,7 +312,7 @@ fn test_dynamic_push2() -> Result<(), Error> { println!("{:x?}", output); println!("{:?}", output); - assert_eq!(output, hex!("61010058")); + assert_eq!(output, hex!("61010158")); Ok(()) } From 26ff19df31738c5b2a005715485e0d70a214547b Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 12:55:51 -0300 Subject: [PATCH 39/48] Fix issue with labels + expressions in dynamic push --- Cargo.lock | 5 +- etk-asm/Cargo.toml | 17 ++++-- etk-asm/src/asm.rs | 55 ++++++++++++++------ etk-asm/tests/asm.rs | 17 ++++-- etk-asm/tests/asm/variable-push/included.etk | 2 +- etk-asm/tests/asm/variable-push/main.etk | 3 +- etk-asm/tests/asm/variable-push2/main.etk | 3 -- etk-asm/tests/asm/variable-push2/main1.etk | 3 ++ etk-asm/tests/asm/variable-push2/main2.etk | 3 ++ etk-asm/tests/asm/variable-push2/main3.etk | 6 +++ 10 files changed, 81 insertions(+), 33 deletions(-) delete mode 100644 etk-asm/tests/asm/variable-push2/main.etk create mode 100644 etk-asm/tests/asm/variable-push2/main1.etk create mode 100644 etk-asm/tests/asm/variable-push2/main2.etk create mode 100644 etk-asm/tests/asm/variable-push2/main3.etk diff --git a/Cargo.lock b/Cargo.lock index d2a9264e..1f8d0a38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,6 +439,7 @@ dependencies = [ "hex", "hex-literal", "num-bigint", + "num-traits", "pest", "pest_derive", "rand", @@ -936,9 +937,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] diff --git a/etk-asm/Cargo.toml b/etk-asm/Cargo.toml index a479bd3e..e2b4622d 100644 --- a/etk-asm/Cargo.toml +++ b/etk-asm/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "etk-asm" version = "0.4.0-dev" -authors = ["Sam Wilson ", "lightclient "] +authors = [ + "Sam Wilson ", + "lightclient ", +] license = "MIT OR Apache-2.0" edition = "2018" description = "EVM Toolkit assembler" @@ -9,23 +12,29 @@ homepage = "https://quilt.github.io/etk" repository = "https://github.com/quilt/etk" readme = "README.md" keywords = ["etk", "ethereum", "assembler"] -categories = ["cryptography::cryptocurrencies", "command-line-utilities", "development-tools", "compilers"] +categories = [ + "cryptography::cryptocurrencies", + "command-line-utilities", + "development-tools", + "compilers", +] [features] cli = ["clap", "etk-cli"] -backtraces = [ "snafu/backtraces", "etk-ops/backtraces" ] +backtraces = ["snafu/backtraces", "etk-ops/backtraces"] [dependencies] etk-ops = { path = "../etk-ops", version = "0.4.0-dev" } etk-cli = { optional = true, path = "../etk-cli", version = "0.4.0-dev" } hex = "0.4.3" num-bigint = "0.4" +num-traits = "0.2.17" pest = "2.1.3" pest_derive = "2.1" rand = "0.8.5" sha3 = "0.10.1" clap = { optional = true, version = "3.1", features = ["derive"] } -snafu = { version = "0.7.1", default-features = false, features = [ "std" ] } +snafu = { version = "0.7.1", default-features = false, features = ["std"] } [dev-dependencies] assert_matches = "1.5.0" diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 83dccc20..e7a002d4 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -143,9 +143,10 @@ mod error { pub use self::error::Error; use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacro}; use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; +use num_traits::ToPrimitive; use rand::Rng; use std::cmp; -use std::collections::{hash_map, HashMap}; +use std::collections::{hash_map, HashMap, HashSet}; /// An item to be assembled, which can be either an [`AbstractOp`], /// the inclusion of a new scope or a raw byte sequence. @@ -254,7 +255,42 @@ impl Assembler { .clone() .concretize((&self.declared_labels, &self.declared_macros).into()) { - Ok(cop) => cop.assemble(&mut output), + Ok(mut cop) => { + if let AbstractOp::Push(imm) = op { + let exp = imm.tree.eval_with_context( + (&self.declared_labels, &self.declared_macros).into(), + ); + + let labels: HashSet = imm + .tree + .labels(&self.declared_macros) + .unwrap() + .into_iter() + .collect(); + if labels.len() > 0 { + if let Ok(val) = exp { + let extra = val.to_usize().unwrap() / 256; + if extra > 0 { + self.concrete_len += extra as usize; + + for label in labels { + if let Some(position) = + self.declared_labels.get_mut(&label) + { + *position = position.map(|v| v + 1); + } + } + } + } + } + + cop = op + .clone() + .concretize((&self.declared_labels, &self.declared_macros).into()) + .unwrap(); + } + cop.assemble(&mut output) + } Err(ops::Error::ContextIncomplete { source: UnknownLabel { .. }, }) => { @@ -358,22 +394,7 @@ impl Assembler { match rop { RawOp::Op(AbstractOp::Label(label)) => { - let mut dst = 0; - for ul in self.undeclared_labels.iter() { - if (ul.label == label) & ul.dynamic_push { - // Compensation in case label was dynamically pushed. - let mut dst_tmp = (self.concrete_len - ul.position) / 256; - - // Size already accounted for %push(label) was 2. - if dst_tmp > 1 { - dst_tmp -= 1; - } - - dst = cmp::max(dst_tmp, dst); - } - } self.undeclared_labels.retain(|l| l.label != *label); - self.concrete_len += dst; let old = self .declared_labels diff --git a/etk-asm/tests/asm.rs b/etk-asm/tests/asm.rs index c7d659c1..07302f6b 100644 --- a/etk-asm/tests/asm.rs +++ b/etk-asm/tests/asm.rs @@ -299,7 +299,9 @@ fn test_dynamic_push_and_include() -> Result<(), Error> { let mut ingester = Ingest::new(&mut output); ingester.ingest_file(source(&["variable-push", "main.etk"]))?; - assert_eq!(output, hex!("61025758585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585801010101010158585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858015b58")); + let tmp: Vec = output.iter().map(|&num| format!("{:x?}", num)).collect(); + println!("{:?}", tmp.join("")); + assert_eq!(output, hex!("61025758585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585801010101010158585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858585858015b")); Ok(()) } @@ -308,11 +310,18 @@ fn test_dynamic_push_and_include() -> Result<(), Error> { fn test_dynamic_push2() -> Result<(), Error> { let mut output = Vec::new(); let mut ingester = Ingest::new(&mut output); - ingester.ingest_file(source(&["variable-push2", "main.etk"]))?; + ingester.ingest_file(source(&["variable-push2", "main1.etk"]))?; + assert_eq!(output, hex!("61010158")); - println!("{:x?}", output); - println!("{:?}", output); + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + ingester.ingest_file(source(&["variable-push2", "main2.etk"]))?; assert_eq!(output, hex!("61010158")); + let mut output = Vec::new(); + let mut ingester = Ingest::new(&mut output); + ingester.ingest_file(source(&["variable-push2", "main3.etk"]))?; + assert_eq!(output, hex!("610107015801")); + Ok(()) } diff --git a/etk-asm/tests/asm/variable-push/included.etk b/etk-asm/tests/asm/variable-push/included.etk index c55517ab..db4c91e2 100644 --- a/etk-asm/tests/asm/variable-push/included.etk +++ b/etk-asm/tests/asm/variable-push/included.etk @@ -591,4 +591,4 @@ pc pc pc pc -add +add \ No newline at end of file diff --git a/etk-asm/tests/asm/variable-push/main.etk b/etk-asm/tests/asm/variable-push/main.etk index cef9faf7..08144f7c 100644 --- a/etk-asm/tests/asm/variable-push/main.etk +++ b/etk-asm/tests/asm/variable-push/main.etk @@ -3,5 +3,4 @@ pc pc %include("included.etk") label: -jumpdest -pc \ No newline at end of file +jumpdest \ No newline at end of file diff --git a/etk-asm/tests/asm/variable-push2/main.etk b/etk-asm/tests/asm/variable-push2/main.etk deleted file mode 100644 index 618285ee..00000000 --- a/etk-asm/tests/asm/variable-push2/main.etk +++ /dev/null @@ -1,3 +0,0 @@ -%push(label + 254) -label: -pc \ No newline at end of file diff --git a/etk-asm/tests/asm/variable-push2/main1.etk b/etk-asm/tests/asm/variable-push2/main1.etk new file mode 100644 index 00000000..7c5eb360 --- /dev/null +++ b/etk-asm/tests/asm/variable-push2/main1.etk @@ -0,0 +1,3 @@ +push2 label + 254 +label: +pc \ No newline at end of file diff --git a/etk-asm/tests/asm/variable-push2/main2.etk b/etk-asm/tests/asm/variable-push2/main2.etk new file mode 100644 index 00000000..10fdca9d --- /dev/null +++ b/etk-asm/tests/asm/variable-push2/main2.etk @@ -0,0 +1,3 @@ +%push(label + 254) +label: +pc \ No newline at end of file diff --git a/etk-asm/tests/asm/variable-push2/main3.etk b/etk-asm/tests/asm/variable-push2/main3.etk new file mode 100644 index 00000000..7c1f1d0e --- /dev/null +++ b/etk-asm/tests/asm/variable-push2/main3.etk @@ -0,0 +1,6 @@ +%push(label1 + label2 + 254) +add +label1: + pc +label2: + add \ No newline at end of file From e904798333576cb65b1306897afc90ffc15ffdb1 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 15:27:43 -0300 Subject: [PATCH 40/48] Fix issue with labels + expressions in dynamic push (rustfmt) --- etk-asm/src/asm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index e7a002d4..8ae6ba2e 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -267,11 +267,11 @@ impl Assembler { .unwrap() .into_iter() .collect(); - if labels.len() > 0 { + if !labels.is_empty() { if let Ok(val) = exp { let extra = val.to_usize().unwrap() / 256; if extra > 0 { - self.concrete_len += extra as usize; + self.concrete_len += extra; for label in labels { if let Some(position) = From 8cc1b5fc6beeae309648b3f39a4fdb24889e7326 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 16:48:24 -0300 Subject: [PATCH 41/48] Minor changes --- etk-asm/src/asm.rs | 63 ++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 8ae6ba2e..3347768c 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -145,7 +145,6 @@ use crate::ops::expression::Error::{UndefinedVariable, UnknownLabel, UnknownMacr use crate::ops::{self, AbstractOp, Assemble, Expression, MacroDefinition}; use num_traits::ToPrimitive; use rand::Rng; -use std::cmp; use std::collections::{hash_map, HashMap, HashSet}; /// An item to be assembled, which can be either an [`AbstractOp`], @@ -175,6 +174,12 @@ impl From> for RawOp { } } +impl Into for &AbstractOp { + fn into(self) -> RawOp { + RawOp::Op(self.clone()) + } +} + /// Assembles a series of [`RawOp`] into raw bytes, tracking and resolving macros and labels, /// and handling dynamic pushes. /// @@ -217,12 +222,6 @@ pub struct Assembler { struct PendingLabel { /// The name of the label. label: String, - - /// Position where the label was invoked. - position: usize, - - /// Whether the label was invoked with a dynamic push or not. - dynamic_push: bool, } impl Assembler { @@ -234,10 +233,10 @@ impl Assembler { /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. fn inspect_macros(&mut self, nodes: &[O]) -> Result<(), Error> where - O: Into + Clone, //TODO: Remove clone and fix code + O: Into + Clone, { for op in nodes.iter() { - let raw_op: RawOp = (*op).clone().into(); + let raw_op: RawOp = op.clone().into(); if let RawOp::Op(AbstractOp::MacroDefinition(_)) = raw_op { self.declare_macro(raw_op)? } @@ -347,12 +346,12 @@ impl Assembler { /// Returns the code of the assembled program. pub fn assemble(&mut self, ops: &[O]) -> Result, Error> where - O: Into + Clone, //TODO: Remove clone and fix code + O: Into + Clone, { self.inspect_macros(ops)?; for op in ops { - self.push(op)?; + self.push(op.clone().into())?; } self.finish()?; @@ -363,11 +362,8 @@ impl Assembler { } /// Insert explicitly declared macros, via `AbstractOp`, into the `Assembler`. - fn declare_macro(&mut self, rop: O) -> Result<(), Error> - where - O: Into, - { - let rop = rop.into(); + fn declare_macro(&mut self, rop: RawOp) -> Result<(), Error> { + //let rop = rop.into(); if let RawOp::Op(AbstractOp::MacroDefinition(ref defn)) = rop { match self.declared_macros.entry(defn.name().to_owned()) { hash_map::Entry::Occupied(_) => { @@ -383,13 +379,11 @@ impl Assembler { } /// Feed a single instruction into the `Assembler`. - fn push(&mut self, rop: &O) -> Result + fn push(&mut self, rop: O) -> Result where - O: Into + Clone, //TODO: Remove clone and fix code + O: Into, { - //let rop = rop.into(); - let rop = rop.clone().into(); - + let rop = rop.into(); self.declare_label(&rop)?; match rop { @@ -413,7 +407,7 @@ impl Assembler { { Ok(cop) => { self.concrete_len += cop.size(); - self.ready.push(rop) + self.ready.push(rop.clone()) } Err(ops::Error::ExpressionTooLarge { value, spec, .. }) => { return error::ExpressionTooLarge { @@ -433,20 +427,14 @@ impl Assembler { Err(ops::Error::ContextIncomplete { source: UnknownLabel { label, .. }, }) => { - let mut dynamic_push = false; if let AbstractOp::Push(_) = op { - dynamic_push = true; self.concrete_len += 2; } else { self.concrete_len += op.size().unwrap(); } - self.undeclared_labels.push(PendingLabel { - label, - position: self.ready.len(), - dynamic_push, - }); - self.ready.push(rop); + self.undeclared_labels.push(PendingLabel { label }); + self.ready.push(rop.clone()); } Err(ops::Error::ContextIncomplete { source: UnknownMacro { name, .. }, @@ -458,7 +446,7 @@ impl Assembler { } RawOp::Raw(raw) => { self.concrete_len += raw.len(); - self.ready.push(RawOp::Raw(raw)); + self.ready.push(RawOp::Raw(raw.to_vec())); } RawOp::Scope(scope) => { let mut asm = Self::new(); @@ -472,15 +460,14 @@ impl Assembler { } fn declare_label(&mut self, rop: &RawOp) -> Result<(), Error> { - if let RawOp::Op(AbstractOp::Label(ref label)) = *rop { - match self.declared_labels.entry(label.to_owned()) { - hash_map::Entry::Occupied(_) => { - return error::DuplicateLabel { label }.fail(); - } - hash_map::Entry::Vacant(v) => { - v.insert(None); + if let RawOp::Op(AbstractOp::Label(label)) = rop { + if self.declared_labels.contains_key(label) { + return error::DuplicateLabel { + label: label.to_owned(), } + .fail(); } + self.declared_labels.insert(label.to_owned(), None); } Ok(()) } From 62b3510a99182f6fade763dcd88d5ec80a383aab Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 16:52:03 -0300 Subject: [PATCH 42/48] PendingLabel removed --- etk-asm/src/asm.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 3347768c..7e51cd55 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -214,14 +214,7 @@ pub struct Assembler { /// Labels that have been referred to (ex. with push) but /// have not been declared with an `AbstractOp::Label`. - undeclared_labels: Vec, -} - -/// Struct used to keep track of pending label invocations and their positions in code. -#[derive(Debug, Clone)] -struct PendingLabel { - /// The name of the label. - label: String, + undeclared_labels: Vec, } impl Assembler { @@ -296,7 +289,7 @@ impl Assembler { let undeclared_names: Vec<_> = self .undeclared_labels .iter() - .map(|PendingLabel { label, .. }| label.clone()) + .map(|label| label.clone()) .collect(); return error::UndeclaredLabels { labels: undeclared_names, @@ -332,7 +325,7 @@ impl Assembler { labels: self .undeclared_labels .iter() - .map(|l| l.label.to_owned()) + .map(|l| l.to_owned()) .collect::>(), } .fail(); @@ -388,7 +381,7 @@ impl Assembler { match rop { RawOp::Op(AbstractOp::Label(label)) => { - self.undeclared_labels.retain(|l| l.label != *label); + self.undeclared_labels.retain(|l| *l != label); let old = self .declared_labels @@ -433,7 +426,7 @@ impl Assembler { self.concrete_len += op.size().unwrap(); } - self.undeclared_labels.push(PendingLabel { label }); + self.undeclared_labels.push(label); self.ready.push(rop.clone()); } Err(ops::Error::ContextIncomplete { From 02492b268bd8cd94c4fb0bdb143cb09eee0c0e7a Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 17:02:23 -0300 Subject: [PATCH 43/48] Into -> From for &AbstractOp --- etk-asm/src/asm.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 7e51cd55..e523996a 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -174,9 +174,9 @@ impl From> for RawOp { } } -impl Into for &AbstractOp { - fn into(self) -> RawOp { - RawOp::Op(self.clone()) +impl From<&AbstractOp> for RawOp { + fn from(op: &AbstractOp) -> Self { + Self::Op(op.clone()) } } From 4dcfd943148f595328e2372d372f6cbcd0201f72 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 17:19:06 -0300 Subject: [PATCH 44/48] Better naming. Code reorg. --- etk-asm/src/asm.rs | 80 +++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 47 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index e523996a..79fef9c9 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -223,23 +223,20 @@ impl Assembler { Self::default() } - /// Inspect the macros in a series of [`RawOp`] and declare them in the assembler. - fn inspect_macros(&mut self, nodes: &[O]) -> Result<(), Error> - where - O: Into + Clone, - { - for op in nodes.iter() { - let raw_op: RawOp = op.clone().into(); - if let RawOp::Op(AbstractOp::MacroDefinition(_)) = raw_op { - self.declare_macro(raw_op)? + /// Backpatch dynamic operations and emit the assembled program. + /// + /// Errors if there are any undeclared labels. + fn backpatch_and_emit(&mut self) -> Result, Error> { + if !self.undeclared_labels.is_empty() { + return error::UndeclaredLabels { + labels: self + .undeclared_labels + .iter() + .map(|l| l.to_owned()) + .collect::>(), } + .fail(); } - - Ok(()) - } - - /// Concretize all assembled instructions. - fn concretize_ops(&mut self) -> Result, Error> { let mut output = Vec::new(); for op in self.ready.iter() { if let RawOp::Op(ref op) = op { @@ -317,18 +314,25 @@ impl Assembler { Ok(output) } - /// Check if the input sequence is complete. Returns any errors that - /// may remain. - fn finish(&mut self) -> Result<(), Error> { - if !self.undeclared_labels.is_empty() { - return error::UndeclaredLabels { - labels: self - .undeclared_labels - .iter() - .map(|l| l.to_owned()) - .collect::>(), + /// Pre-define macros, via `AbstractOp`, into the `Assembler`. + /// + /// This is used to define macros that are used in the same scope. + fn pre_define_macros(&mut self, ops: &[O]) -> Result<(), Error> + where + O: Into + Clone, + { + for op in ops { + let rop = op.clone().into(); + if let RawOp::Op(AbstractOp::MacroDefinition(ref defn)) = rop { + match self.declared_macros.entry(defn.name().to_owned()) { + hash_map::Entry::Occupied(_) => { + return error::DuplicateMacro { name: defn.name() }.fail() + } + hash_map::Entry::Vacant(v) => { + v.insert(defn.to_owned()); + } + } } - .fail(); } Ok(()) @@ -341,36 +345,18 @@ impl Assembler { where O: Into + Clone, { - self.inspect_macros(ops)?; + //self.inspect_macros(ops)?; + self.pre_define_macros(ops)?; for op in ops { self.push(op.clone().into())?; } - self.finish()?; - - let output = self.concretize_ops()?; + let output = self.backpatch_and_emit()?; self.ready.clear(); Ok(output) } - /// Insert explicitly declared macros, via `AbstractOp`, into the `Assembler`. - fn declare_macro(&mut self, rop: RawOp) -> Result<(), Error> { - //let rop = rop.into(); - if let RawOp::Op(AbstractOp::MacroDefinition(ref defn)) = rop { - match self.declared_macros.entry(defn.name().to_owned()) { - hash_map::Entry::Occupied(_) => { - return error::DuplicateMacro { name: defn.name() }.fail() - } - hash_map::Entry::Vacant(v) => { - v.insert(defn.to_owned()); - } - } - } - - Ok(()) - } - /// Feed a single instruction into the `Assembler`. fn push(&mut self, rop: O) -> Result where From 9fd54c9dd53d3fb6d00e10d6353e430155c3ee28 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 17:24:08 -0300 Subject: [PATCH 45/48] Better naming. Code reorg. --- etk-asm/src/asm.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 79fef9c9..32f9cdfe 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -317,7 +317,7 @@ impl Assembler { /// Pre-define macros, via `AbstractOp`, into the `Assembler`. /// /// This is used to define macros that are used in the same scope. - fn pre_define_macros(&mut self, ops: &[O]) -> Result<(), Error> + fn declare_macros(&mut self, ops: &[O]) -> Result<(), Error> where O: Into + Clone, { @@ -345,8 +345,7 @@ impl Assembler { where O: Into + Clone, { - //self.inspect_macros(ops)?; - self.pre_define_macros(ops)?; + self.declare_macros(ops)?; for op in ops { self.push(op.clone().into())?; From 8296d16f892bf16497ab2c922a2045adfb623800 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 17:47:13 -0300 Subject: [PATCH 46/48] Better naming. --- etk-asm/src/asm.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index 32f9cdfe..d74e39d7 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -283,13 +283,8 @@ impl Assembler { Err(ops::Error::ContextIncomplete { source: UnknownLabel { .. }, }) => { - let undeclared_names: Vec<_> = self - .undeclared_labels - .iter() - .map(|label| label.clone()) - .collect(); return error::UndeclaredLabels { - labels: undeclared_names, + labels: self.undeclared_labels.to_vec(), } .fail(); } From f9bc8ccfd2210a8500f265316cc3123510f3259e Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Thu, 9 Nov 2023 18:17:13 -0300 Subject: [PATCH 47/48] Code reorg. --- etk-asm/src/asm.rs | 194 ++++++++++++++++++++++----------------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/etk-asm/src/asm.rs b/etk-asm/src/asm.rs index d74e39d7..195ee683 100644 --- a/etk-asm/src/asm.rs +++ b/etk-asm/src/asm.rs @@ -223,89 +223,21 @@ impl Assembler { Self::default() } - /// Backpatch dynamic operations and emit the assembled program. + /// Feed instructions into the `Assembler`. /// - /// Errors if there are any undeclared labels. - fn backpatch_and_emit(&mut self) -> Result, Error> { - if !self.undeclared_labels.is_empty() { - return error::UndeclaredLabels { - labels: self - .undeclared_labels - .iter() - .map(|l| l.to_owned()) - .collect::>(), - } - .fail(); - } - let mut output = Vec::new(); - for op in self.ready.iter() { - if let RawOp::Op(ref op) = op { - match op - .clone() - .concretize((&self.declared_labels, &self.declared_macros).into()) - { - Ok(mut cop) => { - if let AbstractOp::Push(imm) = op { - let exp = imm.tree.eval_with_context( - (&self.declared_labels, &self.declared_macros).into(), - ); - - let labels: HashSet = imm - .tree - .labels(&self.declared_macros) - .unwrap() - .into_iter() - .collect(); - if !labels.is_empty() { - if let Ok(val) = exp { - let extra = val.to_usize().unwrap() / 256; - if extra > 0 { - self.concrete_len += extra; - - for label in labels { - if let Some(position) = - self.declared_labels.get_mut(&label) - { - *position = position.map(|v| v + 1); - } - } - } - } - } - - cop = op - .clone() - .concretize((&self.declared_labels, &self.declared_macros).into()) - .unwrap(); - } - cop.assemble(&mut output) - } - Err(ops::Error::ContextIncomplete { - source: UnknownLabel { .. }, - }) => { - return error::UndeclaredLabels { - labels: self.undeclared_labels.to_vec(), - } - .fail(); - } - Err(ops::Error::ContextIncomplete { - source: UnknownMacro { name, .. }, - }) => { - return error::UndeclaredInstructionMacro { name }.fail(); - } - Err(ops::Error::ContextIncomplete { - source: UndefinedVariable { name, .. }, - }) => { - return error::UndeclaredVariableMacro { var: name }.fail(); - } + /// Returns the code of the assembled program. + pub fn assemble(&mut self, ops: &[O]) -> Result, Error> + where + O: Into + Clone, + { + self.declare_macros(ops)?; - Err(_) => unreachable!("all ops should be concretizable"), - } - } else if let RawOp::Raw(raw) = op { - output.extend(raw); - } + for op in ops { + self.push(op.clone().into())?; } + let output = self.backpatch_and_emit()?; + self.ready.clear(); Ok(output) } @@ -333,24 +265,6 @@ impl Assembler { Ok(()) } - /// Feed instructions into the `Assembler`. - /// - /// Returns the code of the assembled program. - pub fn assemble(&mut self, ops: &[O]) -> Result, Error> - where - O: Into + Clone, - { - self.declare_macros(ops)?; - - for op in ops { - self.push(op.clone().into())?; - } - - let output = self.backpatch_and_emit()?; - self.ready.clear(); - Ok(output) - } - /// Feed a single instruction into the `Assembler`. fn push(&mut self, rop: O) -> Result where @@ -432,6 +346,92 @@ impl Assembler { Ok(self.concrete_len) } + /// Backpatch dynamic operations and emit the assembled program. + /// + /// Errors if there are any undeclared labels. + fn backpatch_and_emit(&mut self) -> Result, Error> { + if !self.undeclared_labels.is_empty() { + return error::UndeclaredLabels { + labels: self + .undeclared_labels + .iter() + .map(|l| l.to_owned()) + .collect::>(), + } + .fail(); + } + let mut output = Vec::new(); + for op in self.ready.iter() { + if let RawOp::Op(ref op) = op { + match op + .clone() + .concretize((&self.declared_labels, &self.declared_macros).into()) + { + Ok(mut cop) => { + if let AbstractOp::Push(imm) = op { + let exp = imm.tree.eval_with_context( + (&self.declared_labels, &self.declared_macros).into(), + ); + + let labels: HashSet = imm + .tree + .labels(&self.declared_macros) + .unwrap() + .into_iter() + .collect(); + if !labels.is_empty() { + if let Ok(val) = exp { + let extra = val.to_usize().unwrap() / 256; + if extra > 0 { + self.concrete_len += extra; + + for label in labels { + if let Some(position) = + self.declared_labels.get_mut(&label) + { + *position = position.map(|v| v + 1); + } + } + } + } + } + + cop = op + .clone() + .concretize((&self.declared_labels, &self.declared_macros).into()) + .unwrap(); + } + cop.assemble(&mut output) + } + Err(ops::Error::ContextIncomplete { + source: UnknownLabel { .. }, + }) => { + return error::UndeclaredLabels { + labels: self.undeclared_labels.to_vec(), + } + .fail(); + } + Err(ops::Error::ContextIncomplete { + source: UnknownMacro { name, .. }, + }) => { + return error::UndeclaredInstructionMacro { name }.fail(); + } + Err(ops::Error::ContextIncomplete { + source: UndefinedVariable { name, .. }, + }) => { + return error::UndeclaredVariableMacro { var: name }.fail(); + } + + Err(_) => unreachable!("all ops should be concretizable"), + } + } else if let RawOp::Raw(raw) = op { + output.extend(raw); + } + } + + Ok(output) + } + fn declare_label(&mut self, rop: &RawOp) -> Result<(), Error> { if let RawOp::Op(AbstractOp::Label(label)) = rop { if self.declared_labels.contains_key(label) { From 74ebab37975db34d42451c1d03e429c9e1aed235 Mon Sep 17 00:00:00 2001 From: Gaston Zanitti Date: Fri, 10 Nov 2023 12:43:03 -0300 Subject: [PATCH 48/48] Node::Raw added --- etk-asm/src/ast.rs | 1 + etk-asm/src/ingest.rs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/etk-asm/src/ast.rs b/etk-asm/src/ast.rs index e3824c15..ccdf9f16 100644 --- a/etk-asm/src/ast.rs +++ b/etk-asm/src/ast.rs @@ -8,6 +8,7 @@ pub(crate) enum Node { Op(AbstractOp), Import(PathBuf), Include(PathBuf), + Raw(Vec), IncludeHex(PathBuf), } impl From> for Node { diff --git a/etk-asm/src/ingest.rs b/etk-asm/src/ingest.rs index 17ff2b75..d0b381af 100644 --- a/etk-asm/src/ingest.rs +++ b/etk-asm/src/ingest.rs @@ -367,7 +367,6 @@ mod tests { use std::fmt::Display; use std::io::Write; - use std::path; use super::*;