RFC0020 implementation

apps/interpreter/test/assets/broken-model.jv

@@ -28,27 +28,50 @@ pipeline CarsPipeline {
 		write: ["name"];
 	}
 
+  valuetype Car {
+    property name oftype text;
+    property mpg oftype decimal;
+    property cyl oftype integer;
+    property disp oftype decimal;
+    property hp oftype integer;
+    property drat oftype decimal;
+    property wt oftype decimal;
+    property qsec oftype decimal;
+    property vs oftype integer;
+    property am oftype integer;
+    property gear oftype integer;
+    property carb oftype integer;
+  }
+
+  transform CarParser {
+    from r oftype Collection<text>;
+    to car oftype Car;
+
+    car: {
+      name: asText (r cellInColumn "name"),
+      mpg: asDecimal (r cellInColumn "mpg"),
+      cyl: asInteger (r cellInColumn 2),
+      disp: asDecimal (r cellInColumn 3),
+      hp: asInteger (r cellInColumn "hp"),
+      drat: asDecimal (r cellInColumn "drat"),
+      wt: asDecimal (r cellInColumn "wt"),
+      qsec: asDecimal (r cellInColumn "qsec"),
+      vs: asInteger (r cellInColumn "vs"),
+      am: asInteger (r cellInColumn "am"),
+      gear: asInteger (r cellInColumn "gear"),
+      carb: asInteger (r cellInColumn "carb")
+    };
+  }
+
 	block CarsTableInterpreter oftype TableInterpreter {
 		header: true;
-		columns: [
-			"name" oftype text,
-			"mpg" oftype decimal,
-			"cyl" oftype integer,
-			"disp" oftype decimal,
-			"hp" oftype integer,
-			"drat" oftype decimal,
-			"wt" oftype decimal,
-			"qsec" oftype decimal,
-			"vs" oftype integer,
-			"am" oftype integer,
-			"gear" oftype integer,
-			"carb" oftype integer
-		];
+		columns: Car;
+    parseWith: CarParser;
 	}
 
 	block CarsLoader oftype SQLiteLoader {
 		table: "Cars";
 		file: "./cars.sqlite";
 	}
 
-}
+}

example/cars.jv

@@ -7,23 +7,21 @@
 // - Understand the core concepts pipeline, block, and pipe
 // - Understand the general structure of a pipeline
 
-// 1. This Jayvee model describes a pipeline 
-// from a CSV file in the web 
-// to a SQLite file sink.
+// 1. This Jayvee model describes a pipeline from a CSV file in the web to a
+// SQLite file sink.
 pipeline CarsPipeline {
 
-  // 2. We describe the structure of the pipeline,
-  // usually at the top of the pipeline.
-  // by connecting blocks via pipes. 
+  // 2. We describe the structure of the pipeline, usually at the top of the
+  // pipeline, by connecting blocks via pipes.
 
   // 3. Syntax of a pipe
   // connecting the block CarsExtractor
   // with the block CarsTextFileInterpreter.
   CarsExtractor
     -> CarsTextFileInterpreter;
 
-  // 4. The output of the preceding block is hereby used 
-  // as input for the succeeding block.
+  // 4. The output of the preceding block is hereby used as input for the
+  // succeeding block.
 
   // 5. Pipes can be further chained, 
   // leading to an overview of the pipeline.
@@ -34,74 +32,100 @@ pipeline CarsPipeline {
     -> CarsLoader;
 
 
-  // 6. Below the pipes, we usually define the blocks 
-  // that are connected by the pipes.
+  // 6. Below the pipes, we usually define the blocks that are connected by the
+  // pipes.
 
   // 7. Blocks instantiate a block type by using the oftype keyword.
-  // The block type defines the available properties that the block
-  // can use to specify the intended behavior of the block 
+  // The block type defines the available properties that can be used to specify
+  // the intended behavior of the block.
   block CarsExtractor oftype HttpExtractor {
 
     // 8. Properties are assigned to concrete values.
     // Here, we specify the URL where the file shall be downloaded from.
     url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv";
   }
 
-  // 9. The HttpExtractor requires no input and produces a binary file as output.
-  // This file has to be interpreted, e.g., as text file.
+  // 9. The HttpExtractor requires no input and produces a binary file as
+  // output. This file has to be interpreted, e.g., as text file.
   block CarsTextFileInterpreter oftype TextFileInterpreter { }
 
   // 10. Next, we interpret the text file as sheet.
-  // A sheet only contains text cells and is useful for manipulating the shape of data before assigning more strict value types to cells.
+  // A sheet only contains text cells and is useful for manipulating the shape
+  // of data before assigning more strict value types to cells.
   block CarsCSVInterpreter oftype CSVInterpreter {
     enclosing: '"';
   }
 
   // 11. We can write into cells of a sheet using the CellWriter block type.
   block NameHeaderWriter oftype CellWriter {
     // 12. We utilize a syntax similar to spreadsheet programs.
-    // Cell ranges can be described using the keywords "cell", "row", "column", or "range" that indicate which 
-    // cells are selected for the write action.
+    // Cell ranges can be described using the keywords "cell", "row", "column",
+    // or "range" that indicate which cells are selected for the write action.
     at: cell A1;
 
-    // 13. For each cell we selected with the "at" property above,
-    // we can specify what value shall be written into the cell.
+    // 13. For each cell we selected with the "at" property above, we can
+    // specify what value shall be written into the cell.
     write: [
       "name"
     ];
   }
 
-  // 14. As a next step, we interpret the sheet as a table by adding structure.
-  // We define a value type per column that specifies the data type of the column.
-  // Rows that include values that are not valid according to the their value types are dropped automatically. 
+  // 14. Next, we define the schema of our table.
+  // For this, we use a value type where each property corresponds to a column
+  // in the table. The column will have the same valuetype as the property.
+  valuetype Car {
+    property name oftype text;
+    property mpg oftype decimal;
+    property cyl oftype integer;
+    property disp oftype decimal;
+    property hp oftype integer;
+    property drat oftype decimal;
+    property wt oftype decimal;
+    property qsec oftype decimal;
+    property vs oftype integer;
+    property am oftype integer;
+    property gear oftype integer;
+    property carb oftype integer;
+  }
+
+  transform CarParser {
+    from r oftype Collection<text>;
+    to car oftype Car;
+
+    car: {
+      name: asText (r cellInColumn "name"),
+      mpg: asDecimal (r cellInColumn "mpg"),
+      cyl: asInteger (r cellInColumn 2),
+      disp: asDecimal (r cellInColumn 3),
+      hp: asInteger (r cellInColumn "hp"),
+      drat: asDecimal (r cellInColumn "drat"),
+      wt: asDecimal (r cellInColumn "wt"),
+      qsec: asDecimal (r cellInColumn "qsec"),
+      vs: asInteger (r cellInColumn "vs"),
+      am: asInteger (r cellInColumn "am"),
+      gear: asInteger (r cellInColumn "gear"),
+      carb: asInteger (r cellInColumn "carb")
+    };
+  }
+
+  // 15. As a next step, we interpret the sheet as a table, using the valuetype
+  // defined above. Rows that include values that are not valid according to the
+  // their value types are dropped automatically.
   block CarsTableInterpreter oftype TableInterpreter {
     header: true;
-    columns: [
-      "name" oftype text,
-      "mpg" oftype decimal,
-      "cyl" oftype integer,
-      "disp" oftype decimal,
-      "hp" oftype integer,
-      "drat" oftype decimal,
-      "wt" oftype decimal,
-      "qsec" oftype decimal,
-      "vs" oftype integer,
-      "am" oftype integer,
-      "gear" oftype integer,
-      "carb" oftype integer
-    ];
+    columns: Car;
+    parseWith: CarParser;
   }
 
-  // 15. As a last step, we load the table into a sink,
-  // here into a sqlite file.
-  // The structural information of the table is used
-  // to generate the correct table.
+  // 16. As a last step, we load the table into a sink, here into a sqlite file.
+  // The structural information of the table is used to generate the correct
+  // table.
   block CarsLoader oftype SQLiteLoader {
     table: "Cars";
     file: "./cars.sqlite";
   }
 
-  // 16. Congratulations!
-  // You can now use the sink for your data analysis, app, 
-  // or whatever you want to do with the cleaned data.  
-}
+  // 17. Congratulations!
+  // You can now use the sink for your data analysis, app, or whatever you want
+  // to do with the cleaned data.
+}

example/electric-vehicles.jv

@@ -49,30 +49,56 @@ pipeline ElectricVehiclesPipeline {
 
   block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { }
 
+  valuetype ElectricVehicle {
+    property vin oftype VehicleIdentificationNumber10;
+    property county oftype text;
+    property city oftype text;
+    property state oftype UsStateCode;
+    property postal oftype text;
+    property modelYear oftype integer;
+    property make oftype text;
+    property model oftype text;
+    property evType oftype text;
+    property cafvEligibility oftype text;
+    property electricRange oftype integer;
+    property baseMSRP oftype integer;
+    property legislativeDistrict oftype text;
+    property dolID oftype integer;
+    property location oftype text;
+    property utility oftype text;
+    property censusTract oftype text;
+  }
+
+  transform ElectricVehicleParser {
+    from r oftype Collection<text>;
+    to ev oftype ElectricVehicle;
+
+    ev: {
+      vin: r cellInColumn "VIN (1-10)",
+      county: asText (r cellInColumn "County"),
+      city: asText (r cellInColumn "City"),
+      state: asText (r cellInColumn "State"),
+      postal: asText (r cellInColumn "Postal Code"),
+      modelYear: asInteger (r cellInColumn "Model Year"),
+      make: asText (r cellInColumn "Make"),
+      model: asText (r cellInColumn "Model"),
+      evType: asText (r cellInColumn "Electric Vehicle Type"),
+      cafvEligibility: asText (r cellInColumn "Clean Alternative Fuel Vehicle (CAFV) Eligibility"),
+      electricRange: asInteger (r cellInColumn "Electric Range"),
+      baseMSRP: asInteger (r cellInColumn "Base MSRP"),
+      legislativeDistrict: asText (r cellInColumn "LegislativeDistrict"),
+      dolID: asInteger (r cellInColumn "DOL Vehicle ID"),
+      location: asText (r cellInColumn "Vehicle Location"),
+      utility: asText (r cellInColumn "Electric Utility"),
+      censusTract: asText (r cellInColumn "2020 Census Tract"),
+    };
+  }
+
+
   block ElectricVehiclesTableInterpreter oftype TableInterpreter {
     header: true;
-    columns: [
-      // 4. Here, a user-deifned value type is used to describe this column.
-      // The capital letter indicates that the value type is not built-in
-      // by convention. The value type itself is defined further below.
-      "VIN (1-10)" oftype VehicleIdentificationNumber10,
-      "County" oftype text,
-      "City" oftype text,
-      "State" oftype UsStateCode, // We can just use the element as if it was defined in this file.
-      "Postal Code" oftype text,
-      "Model Year" oftype integer,
-      "Make" oftype text,
-      "Model" oftype text,
-      "Electric Vehicle Type" oftype text,
-      "Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text,
-      "Electric Range" oftype integer,
-      "Base MSRP" oftype integer,
-      "Legislative District" oftype text,
-      "DOL Vehicle ID" oftype integer,
-      "Vehicle Location" oftype text,
-      "Electric Utility" oftype text,
-      "2020 Census Tract" oftype text,
-    ];
+    columns: ElectricVehicle;
+    parseWith: ElectricVehicleParser;
   }
 
   // 5. This block describes the application of a transform function
@@ -81,9 +107,9 @@ pipeline ElectricVehiclesPipeline {
   // by the "use" property.
   block ElectricRangeTransformer oftype TableTransformer {
     inputColumns: [
-      "Electric Range"
+      "electricRange"
     ];
-    outputColumn: "Electric Range (km)";
+    outputColumn: "electricRange (km)";
     uses: MilesToKilometers;
   }