feat: lazy load fn nodes

core/engine/src/decision_graph/cleaner.rs

@@ -0,0 +1,55 @@
+use ahash::{HashSet, HashSetExt};
+use std::rc::Rc;
+use zen_types::variable::Variable;
+
+const RESERVED_KEYS: &[&str] = &["$nodes"];
+
+pub(crate) struct VariableCleaner {
+    visited: HashSet<usize>,
+}
+
+impl VariableCleaner {
+    pub fn new() -> Self {
+        Self {
+            visited: HashSet::new(),
+        }
+    }
+
+    pub fn clean(&mut self, var: &Variable) {
+        match var {
+            Variable::Null
+            | Variable::Bool(_)
+            | Variable::Number(_)
+            | Variable::String(_)
+            | Variable::Dynamic(_) => {}
+
+            Variable::Array(arr) => {
+                let ptr = Rc::as_ptr(arr) as usize;
+                if !self.visited.insert(ptr) {
+                    return;
+                }
+
+                let items = arr.borrow();
+                for item in items.iter() {
+                    self.clean(item);
+                }
+            }
+
+            Variable::Object(obj) => {
+                let ptr = Rc::as_ptr(obj) as usize;
+                if !self.visited.insert(ptr) {
+                    return;
+                }
+
+                let mut map = obj.borrow_mut();
+                for key in RESERVED_KEYS {
+                    map.remove(*key);
+                }
+
+                for (_, value) in map.iter() {
+                    self.clean(value);
+                }
+            }
+        }
+    }
+}

core/engine/src/decision_graph/graph.rs

@@ -1,3 +1,4 @@
+use crate::decision_graph::cleaner::VariableCleaner;
 use crate::decision_graph::tracer::NodeTracer;
 use crate::decision_graph::walker::{GraphWalker, NodeData, StableDiDecisionGraph};
 use crate::engine::EvaluationTraceKind;
@@ -17,6 +18,7 @@ use crate::nodes::{
 use crate::{DecisionGraphTrace, DecisionGraphValidationError, EvaluationError};
 use ahash::{HashMap, HashMapExt};
 use petgraph::algo::is_cyclic_directed;
+use petgraph::matrix_graph::Zero;
 use serde::ser::SerializeMap;
 use serde::{Deserialize, Serialize, Serializer};
 use std::cell::RefCell;
@@ -189,8 +191,6 @@ impl DecisionGraph {
                 }
             };
 
-            output.dot_remove("$nodes");
-
             walker.set_node_data(
                 nid,
                 NodeData {
@@ -205,10 +205,27 @@ impl DecisionGraph {
             }
         }
 
+        let result = walker.ending_variables(&self.graph);
+        let trace = tracer.into_traces();
+
+        if self.config.iteration.is_zero() {
+            let mut cleaner = VariableCleaner::new();
+            cleaner.clean(&result);
+            if let Some(t) = &trace {
+                t.values().for_each(|v| {
+                    cleaner.clean(&v.input);
+                    cleaner.clean(&v.output);
+                    if let Some(td) = &v.trace_data {
+                        cleaner.clean(td);
+                    }
+                })
+            }
+        }
+
         Ok(DecisionGraphResponse {
-            result: walker.ending_variables(&self.graph),
             performance: format!("{:.1?}", root_start.elapsed()),
-            trace: tracer.into_traces(),
+            result,
+            trace,
         })
     }
 }

core/engine/src/decision_graph/mod.rs

@@ -1,3 +1,4 @@
+pub(crate) mod cleaner;
 mod error;
 pub(crate) mod graph;
 mod tracer;

core/engine/src/nodes/function/v2/function.rs

@@ -6,7 +6,7 @@ use crate::nodes::function::v2::error::{FunctionError, FunctionResult, ResultExt
 use crate::nodes::function::v2::listener::{RuntimeEvent, RuntimeListener};
 use crate::nodes::function::v2::module::console::{Console, Log};
 use crate::nodes::function::v2::module::ModuleLoader;
-use crate::nodes::function::v2::serde::JsValue;
+use crate::nodes::function::v2::serde::{JsValue, JsValueWithNodes};
 use rquickjs::promise::MaybePromise;
 use rquickjs::{async_with, AsyncContext, AsyncRuntime, CatchResultExt, Ctx, Module};
 use serde::{Deserialize, Serialize};
@@ -104,7 +104,7 @@ impl Function {
     pub(crate) async fn call_handler(
         &self,
         name: &str,
-        data: JsValue,
+        data: JsValueWithNodes,
     ) -> FunctionResult<HandlerResponse> {
         let k: FunctionResult<HandlerResponse> = async_with!(&self.ctx => |ctx| {
             self.dispatch_event_inner(&ctx, RuntimeEvent::SoftReset).await?;

core/engine/src/nodes/function/v2/mod.rs

@@ -5,10 +5,11 @@ use crate::nodes::definition::NodeHandler;
 use crate::nodes::function::v2::error::{FunctionError, FunctionResult};
 use crate::nodes::function::v2::function::{Function, HandlerResponse};
 use crate::nodes::function::v2::module::console::Log;
-use crate::nodes::function::v2::serde::JsValue;
+use crate::nodes::function::v2::serde::{JsValue, JsValueWithNodes};
 use crate::nodes::result::NodeResult;
 use crate::nodes::{NodeContext, NodeError};
 use ::serde::{Deserialize, Serialize};
+use rquickjs::prelude::Func;
 use rquickjs::{async_with, CatchResultExt, Object};
 use serde_json::json;
 use zen_expression::variable::ToVariable;
@@ -29,9 +30,6 @@ impl NodeHandler for FunctionV2NodeHandler {
 
     async fn handle(&self, ctx: NodeContext<Self::NodeData, Self::TraceData>) -> NodeResult {
         let start = Instant::now();
-        if ctx.node.omit_nodes {
-            ctx.input.dot_remove("$nodes");
-        }
 
         let function = ctx.function_runtime().await?;
         let module_name = function.suggest_module_name(ctx.id.deref(), ctx.node.source.deref());
@@ -62,7 +60,7 @@ impl NodeHandler for FunctionV2NodeHandler {
             .await?;
 
         let response_result = function
-            .call_handler(&module_name, JsValue(ctx.input.clone()))
+            .call_handler(&module_name, JsValueWithNodes(JsValue(ctx.input.clone())))
             .await;
 
         function.runtime().set_interrupt_handler(None).await;
@@ -91,6 +89,15 @@ impl FunctionV2NodeHandler {
 
             ctx.globals().set("config", config).catch(&ctx)?;
 
+            let nodes_data = node_ctx.input.dot("$nodes").unwrap_or_default();
+
+            ctx.globals()
+                .set(
+                    "__getNodesData",
+                    Func::from(move || JsValue(nodes_data.clone())),
+                )
+                .catch(&ctx)?;
+
             Ok(())
         })
         .await
@@ -103,12 +110,6 @@ pub struct FunctionV2Trace {
     pub log: Vec<Log>,
 }
 
-#[derive(Serialize, Deserialize)]
-pub struct FunctionResponse {
-    performance: String,
-    data: Option<HandlerResponse>,
-}
-
 struct FunctionContext<'a> {
     context: &'a NodeContext<FunctionContent, FunctionV2Trace>,
     function: &'a Function,

core/engine/src/nodes/function/v2/serde.rs

@@ -131,3 +131,41 @@ impl<'js> IntoJs<'js> for JsValue {
         Ok(res)
     }
 }
+
+pub struct JsValueWithNodes(pub JsValue);
+
+impl<'js> IntoJs<'js> for JsValueWithNodes {
+    fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result<QValue<'js>> {
+        let base_js = self.0.into_js(ctx)?;
+        if !base_js.is_object() {
+            return Ok(base_js);
+        }
+
+        let obj = base_js.into_object().or_throw(ctx)?;
+        let nodes_proxy: QValue<'js> = ctx.eval(
+            r#"
+            (() => {
+              const _data = { loaded: false, inner: null };
+              const data = () => {
+                if (!_data.loaded) {
+                  _data.loaded = true;
+                  _data.inner = __getNodesData();
+                }
+
+                return _data.inner;
+              };
+
+              return new Proxy({}, {
+                get: (target, prop) => data()[prop],
+                has: (target, prop) => prop in data(),
+                ownKeys: () => Object.keys(data()),
+                getOwnPropertyDescriptor: (target, prop) => Object.getOwnPropertyDescriptor(data(), prop),
+              });
+            })();
+        "#,
+        )?;
+
+        obj.set("$nodes", nodes_proxy)?;
+        Ok(obj.into_value())
+    }
+}

core/engine/src/nodes/transform_attributes.rs

@@ -61,7 +61,6 @@ impl TransformAttributesExecution for TransformAttributes {
                     });
                 }
 
-                response.output.dot_remove("$nodes");
                 response.output
             }
             TransformExecutionMode::Loop => {
@@ -90,7 +89,6 @@ impl TransformAttributesExecution for TransformAttributes {
                         response.output = input.clone().merge_clone(&response.output);
                     }
 
-                    response.output.dot_remove("$nodes");
                     output_array.push(response.output);
                 }
 

core/engine/tests/engine.rs

@@ -9,7 +9,6 @@ use std::sync::Arc;
 use tokio::runtime::Builder;
 use zen_engine::loader::{LoaderError, MemoryLoader};
 use zen_engine::model::{DecisionContent, DecisionNode, DecisionNodeKind, FunctionNodeContent};
-use zen_engine::nodes::NodeError;
 use zen_engine::Variable;
 use zen_engine::{DecisionEngine, EvaluationError, EvaluationOptions};
 
@@ -409,3 +408,24 @@ async fn test_validation() {
         .await
         .is_err());
 }
+
+#[tokio::test]
+#[cfg_attr(miri, ignore)]
+async fn test_nodes_reference() {
+    let engine = DecisionEngine::default().with_loader(Arc::new(create_fs_loader()));
+
+    let evaluation = engine
+        .evaluate("$nodes-parent.json", json!({ "hello": "world" }).into())
+        .await;
+
+    assert!(evaluation.is_ok());
+    assert_eq!(
+        evaluation.unwrap().result.to_value(),
+        json!({
+            "expressionParentNodes": { "request": { "hello": "world" } },
+            "expressionRequest": { "hello": "world" },
+            "functionParentNodes": { "request": { "hello": "world" } },
+            "functionRequest": { "hello": "world" },
+        })
+    );
+}

core/types/src/decision/mod.rs

@@ -94,8 +94,6 @@ pub enum FunctionNodeContent {
 #[serde(rename_all = "camelCase")]
 pub struct FunctionContent {
     pub source: Arc<str>,
-    #[serde(default)]
-    pub omit_nodes: bool,
 }
 
 #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]

test-data/$nodes-child.json

@@ -0,0 +1,69 @@
+{
+  "contentType": "application/vnd.gorules.decision",
+  "nodes": [
+    {
+      "type": "inputNode",
+      "content": {
+        "schema": ""
+      },
+      "id": "adbcb4f4-015c-429f-a9b5-5e08e50470d8",
+      "name": "request",
+      "position": {
+        "x": 80,
+        "y": 230
+      }
+    },
+    {
+      "type": "expressionNode",
+      "content": {
+        "expressions": [
+          {
+            "id": "ffad6138-2b20-4439-b043-b0e835573178",
+            "key": "expressionParentNodes",
+            "value": "$nodes.request.$nodes"
+          },
+          {
+            "id": "ffad6138-2b20-4439-b043-b0e835573178",
+            "key": "expressionRequest",
+            "value": "$nodes.request"
+          }
+        ],
+        "inputField": null,
+        "outputPath": null,
+        "executionMode": "single"
+      },
+      "id": "ec43772d-7c5a-4d36-852b-cc18f470e2f5",
+      "name": "expression1",
+      "position": {
+        "x": 430,
+        "y": 230
+      }
+    },
+    {
+      "type": "functionNode",
+      "content": {
+        "source": "import zen from 'zen';\n\n/** @type {Handler} **/\nexport const handler = async (input) => {\n  return {\n    functionParentNodes: input['$nodes'].request['$nodes']\n, functionRequest: input['$nodes'].request\n  };\n};\n"
+      },
+      "id": "91b84889-a9f7-4452-bea6-8610f6ba4084",
+      "name": "function1",
+      "position": {
+        "x": 430,
+        "y": 135
+      }
+    }
+  ],
+  "edges": [
+    {
+      "id": "06c0b8f4-45a3-4422-b58a-27da004d0786",
+      "sourceId": "adbcb4f4-015c-429f-a9b5-5e08e50470d8",
+      "type": "edge",
+      "targetId": "ec43772d-7c5a-4d36-852b-cc18f470e2f5"
+    },
+    {
+      "id": "5ad36d1a-a4a1-4f89-bec4-ed54175655d0",
+      "sourceId": "adbcb4f4-015c-429f-a9b5-5e08e50470d8",
+      "type": "edge",
+      "targetId": "91b84889-a9f7-4452-bea6-8610f6ba4084"
+    }
+  ]
+}

test-data/$nodes-parent.json

@@ -0,0 +1,34 @@
+{
+  "contentType": "application/vnd.gorules.decision",
+  "edges": [
+    {
+      "id": "0f5ca374-811e-44da-b882-0d5566f43b65",
+      "type": "edge",
+      "sourceId": "341e36a6-be77-44e1-99a5-d7c7ff1b7aba",
+      "targetId": "0b8dcf6b-fc04-47cb-bf82-bda764e6c09b"
+    }
+  ],
+  "nodes": [
+    {
+      "id": "341e36a6-be77-44e1-99a5-d7c7ff1b7aba",
+      "name": "request",
+      "type": "inputNode",
+      "position": {
+        "x": 40,
+        "y": 240
+      }
+    },
+    {
+      "id": "0b8dcf6b-fc04-47cb-bf82-bda764e6c09b",
+      "name": "nodesChild",
+      "type": "decisionNode",
+      "content": {
+        "key": "$nodes-child.json"
+      },
+      "position": {
+        "x": 370,
+        "y": 240
+      }
+    }
+  ]
+}