Add FileTests to JS

build.sc

@@ -1,6 +1,8 @@
-import mill._, scalalib._, publish._, scalajslib._, scalanativelib._, scalanativelib.api._
+import mill._, scalalib._, publish._, scalajslib._, scalanativelib._, scalanativelib.api._, scalajslib.api._
 import $ivy.`com.lihaoyi::mill-contrib-jmh:`
 import contrib.jmh.JmhModule
+import java.util.Base64
+import java.nio.charset.StandardCharsets
 
 val sjsonnetVersion = "0.4.14"
 
@@ -57,17 +59,58 @@ object sjsonnet extends Module {
   trait SjsonnetJsModule extends SjsonnetCrossModule with ScalaJSModule{
     def millSourcePath = super.millSourcePath / os.up
     def scalaJSVersion = "1.17.0"
+    def esVersion = ESVersion.ES2018
     def sources = T.sources(
       this.millSourcePath / "src",
       this.millSourcePath / "src-js",
       this.millSourcePath / "src-jvm-js"
     )
     object test extends ScalaJSTests with CrossTests {
+      def jsEnvConfig = JsEnvConfig.NodeJs(args=List("--stack-size=" + 100 * 1024))
       def sources = T.sources(
         this.millSourcePath / "src",
         this.millSourcePath / "src-js",
         this.millSourcePath / "src-jvm-js"
       )
+      def generatedSources = T{
+        val files = os.walk(this.millSourcePath / "resources").filterNot(os.isDir).map(p => p.relativeTo(this.millSourcePath / "resources") -> os.read.bytes(p)).toMap
+        os.write(
+          T.ctx().dest / "TestResources.scala",
+          s"""package sjsonnet
+             |
+             |object TestResources{
+             |  val files = Map(
+             |""".stripMargin)
+        for((k, v) <- files) {
+          val name = k.toString.replaceAll("/", "_").replaceAll("\\.", "_").replaceAll("-", "_")
+          val values = Base64.getEncoder().encodeToString(v).grouped(65535).toSeq
+          os.write(
+            T.ctx().dest / s"$name.scala",
+            s"""package sjsonnet
+               |
+               |import java.util.Base64
+               |
+               |object $name {
+               |  def contentArr = Seq(
+               |    ${values.map("\"" + _ + "\"").mkString(",\n    ")}
+               |  )
+               |  def content = Base64.getDecoder().decode(contentArr.mkString)
+               |}
+               |""".stripMargin)
+          os.write.append(
+            T.ctx().dest / "TestResources.scala",
+            s"""    "$k" -> $name.content,
+               |""".stripMargin
+          )
+        }
+        os.write.append(
+          T.ctx().dest / "TestResources.scala",
+          s"""  )
+             |}
+             |""".stripMargin
+        )
+        Seq(PathRef(T.ctx().dest / "TestResources.scala")) ++ files.keys.map(p => PathRef(T.ctx().dest / s"${p.toString.replaceAll("/", "_").replaceAll("\\.", "_").replaceAll("-", "_")}.scala"))
+      }
     }
   }
 

sjsonnet/src-js/sjsonnet/SjsonnetMain.scala

@@ -12,7 +12,7 @@ object SjsonnetMain {
                 tlaVars: js.Any,
                 wd0: String,
                 importResolver: js.Function2[String, String, String],
-                importLoader: js.Function1[String, String],
+                importLoader: js.Function2[String, Boolean, Either[String, Array[Byte]]],
                 preserveOrder: Boolean = false): js.Any = {
     val interp = new Interpreter(
       ujson.WebJson.transform(extVars, ujson.Value).obj.toMap.map{case (k, ujson.Str(v)) => (k, v)},
@@ -25,7 +25,10 @@ object SjsonnetMain {
             case s => Some(JsVirtualPath(s))
           }
         def read(path: Path, binaryData: Boolean): Option[ResolvedFile] =
-          Option(StaticResolvedFile(importLoader(path.asInstanceOf[JsVirtualPath].path)))
+          importLoader(path.asInstanceOf[JsVirtualPath].path, binaryData) match {
+            case Left(s) => Some(StaticResolvedFile(s))
+            case Right(arr) => Some(StaticBinaryResolvedFile(arr))
+          }
       },
       parseCache = new DefaultParseCache,
       new Settings(preserveOrder = preserveOrder),

sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala

@@ -26,11 +26,13 @@ class CachedResolvedFile(val resolvedImportPath: OsPath, memoryLimitBytes: Long,
   // Assert that the file is less than limit
   assert(jFile.length() <= memoryLimitBytes, s"Resolved import path $resolvedImportPath is too large: ${jFile.length()} bytes > ${memoryLimitBytes} bytes")
 
-  private[this] val resolvedImportContent: StaticResolvedFile = {
+  private[this] val resolvedImportContent: ResolvedFile = {
     // TODO: Support caching binary data
-    if (jFile.length() > cacheThresholdBytes || binaryData) {
+    if (jFile.length() > cacheThresholdBytes) {
       // If the file is too large, then we will just read it from disk
       null
+    } else if (binaryData) {
+      StaticBinaryResolvedFile(readRawBytes(jFile))
     } else {
       StaticResolvedFile(readString(jFile))
     }

sjsonnet/src/sjsonnet/Importer.scala

@@ -164,6 +164,17 @@ case class StaticResolvedFile(content: String) extends ResolvedFile {
   override def readRawBytes(): Array[Byte] = content.getBytes(StandardCharsets.UTF_8)
 }
 
+case class StaticBinaryResolvedFile(content: Array[Byte]) extends ResolvedFile {
+  def getParserInput(): ParserInput = ??? // Not used for binary imports
+
+  def readString(): String = ??? // Not used for binary imports
+
+  // We just cheat, the content hash can be the content itself for static imports
+  lazy val contentHash: String = content.hashCode().toString
+
+  override def readRawBytes(): Array[Byte] = content
+}
+
 class CachedImporter(parent: Importer) extends Importer {
   val cache = mutable.HashMap.empty[Path, ResolvedFile]
 

sjsonnet/test/resources/test_suite/regex_js.jsonnet

@@ -0,0 +1,72 @@
+std.assertEqual(std.native('regexFullMatch')(@'e', 'hello'), null) &&
+
+std.assertEqual(
+  std.native('regexFullMatch')(@'h.*o', 'hello'),
+  {
+    string: 'hello',
+    captures: [],
+    namedCaptures: {},
+  }
+) &&
+
+std.assertEqual(
+  std.native('regexFullMatch')(@'h(.*)o', 'hello'),
+  {
+    string: 'hello',
+    captures: ['ell'],
+    namedCaptures: {},
+  }
+) &&
+
+std.assertEqual(
+  std.native('regexFullMatch')(@'h(?P<mid>.*)o', 'hello'),
+  {
+    string: 'hello',
+    captures: ['ell'],
+    namedCaptures: {
+      mid: 'ell',
+    },
+  }
+) &&
+
+std.assertEqual(std.native('regexPartialMatch')(@'world', 'hello'), null) &&
+
+std.assertEqual(
+  std.native('regexPartialMatch')(@'e', 'hello'),
+  {
+    string: 'hello',
+    captures: [],
+    namedCaptures: {},
+  }
+) &&
+
+std.assertEqual(
+  std.native('regexPartialMatch')(@'e(.*)o', 'hello'),
+  {
+    string: 'hello',
+    captures: ['ll'],
+    namedCaptures: {},
+  }
+) &&
+
+std.assertEqual(
+  std.native('regexPartialMatch')(@'e(?P<mid>.*)o', 'hello'),
+  {
+    string: 'hello',
+    captures: ['ll'],
+    namedCaptures: {
+      mid: 'll',
+    },
+  }
+) &&
+
+std.assertEqual(std.native('regexQuoteMeta')(@'1.5-2.0?'), '\\Q1.5-2.0?\\E') &&
+
+
+std.assertEqual(std.native('regexReplace')('wishyfishyisishy', @'ish', 'and'), 'wandyfishyisishy') &&
+std.assertEqual(std.native('regexReplace')('yabba dabba doo', @'b+', 'd'), 'yada dabba doo') &&
+
+std.assertEqual(std.native('regexGlobalReplace')('wishyfishyisishy', @'ish', 'and'), 'wandyfandyisandy') &&
+std.assertEqual(std.native('regexGlobalReplace')('yabba dabba doo', @'b+', 'd'), 'yada dada doo') &&
+
+true

sjsonnet/test/resources/test_suite/stdlib_native.jsonnet

@@ -483,8 +483,8 @@ std.assertEqual(std.setMember('a', []), false) &&
 std.assertEqual(std.setMember('a', ['b', 'c']), false) &&
 
 (
-  if std.thisFile == '<stdin>' then
-    // This happens when testing the unparser.
+  if std.thisFile == '<stdin>' || std.thisFile == "(memory)" then
+    // This happens when testing the unparser or scala.js
     true
   else
     std.assertEqual(std.thisFile, 'stdlib_native.jsonnet')

sjsonnet/test/src-js/sjsonnet/FileTests.scala

@@ -0,0 +1,97 @@
+package sjsonnet
+
+import java.util.Base64
+import java.nio.charset.StandardCharsets
+import scala.scalajs.js
+import utest._
+
+object FileTests extends TestSuite {
+  def joinPath(a: String, b: String) = {
+    val aStripped = if (a.endsWith("/")) a.substring(0, a.length - 1) else a
+    val bStripped = if (b.startsWith("/")) b.substring(1) else b
+    if (aStripped.isEmpty)
+      bStripped
+    else if (bStripped.isEmpty)
+      aStripped
+    else
+      aStripped + "/" + bStripped
+  }
+
+  def eval(fileName: String) = {
+    SjsonnetMain.interpret(
+      new String(TestResources.files(joinPath("test_suite", fileName)), StandardCharsets.UTF_8),
+      js.Dictionary("var1" -> """"test"""", "var2" -> """local f(a, b) = {[a]: b, "y": 2}; f("x", 1)"""),
+      js.Dictionary("var1" -> """"test"""", "var2" -> """{"x": 1, "y": 2}"""),
+      "test_suite",
+      (wd: String, path: String) => joinPath(wd, path),
+      (path: String, binaryData: Boolean) => if (binaryData) {
+        Right(TestResources.files(joinPath("test_suite", path)))
+      } else {
+        Left(new String(TestResources.files(joinPath("test_suite", path)), StandardCharsets.UTF_8))
+      }
+    )
+  }
+  def check(expected: ujson.Value = ujson.True)(implicit tp: utest.framework.TestPath) = {
+    val res = ujson.WebJson.transform(eval(s"${tp.value.last}.jsonnet"), ujson.Value)
+    assert(res == expected)
+    res
+  }
+  def checkFail(expected: String)(implicit tp: utest.framework.TestPath) = {
+    try {
+      eval(s"test_suite/${tp.value.last}.jsonnet").asInstanceOf[String]
+      assert(false)
+    } catch {
+      case e: js.JavaScriptException =>
+        assert(e.getMessage == expected)
+    }
+  }
+  def checkGolden()(implicit tp: utest.framework.TestPath) = {
+    check(ujson.read(new String(TestResources.files(joinPath("test_suite", s"${tp.value.last}.jsonnet.golden")), StandardCharsets.UTF_8)))
+  }
+
+  def tests = Tests{
+    test("arith_bool") - check()
+    test("arith_float") - check()
+    test("arith_string") - check()
+    test("array") - check()
+    test("assert") - check()
+    test("binary") - check()
+    test("comments") - check()
+    test("condition") - check()
+    // test("dos_line_endings") - checkGolden()
+    test("format") - check()
+    // test("formatter") - checkGolden()
+    test("formatting_braces") - checkGolden()
+    test("formatting_braces2") - checkGolden()
+    test("functions") - check()
+    test("import") - check()
+    test("invariant") - check()
+    test("invariant_manifest") - checkGolden()
+    test("local") - check()
+    test("merge") - check()
+    test("null") - check()
+    test("object") - check()
+    test("oop") - check()
+    test("oop_extra") - check()
+    test("parsing_edge_cases") - check()
+    test("precedence") - check()
+    test("recursive_function_native") - check()
+    test("recursive_import_ok") - check()
+    test("recursive_object") - check()
+    test("regex_js") - check()
+    test("sanity") - checkGolden()
+    test("sanity2") - checkGolden()
+    test("shebang") - check()
+    test("slice.sugar") - check()
+    test("std_all_hidden") - check()
+    test("stdlib_native") - check()
+    test("text_block") - check()
+    test("tla.simple")- check()
+    test("unicode") - check()
+    test("unix_line_endings") - checkGolden()
+    test("unparse") - checkGolden()
+    test("verbatim_strings") - check()
+    test("issue_127") - check()
+  }
+}
+

sjsonnet/test/src-native/sjsonnet/FileTests.scala

@@ -56,6 +56,7 @@ object FileTests extends TestSuite{
     test("recursive_function_native") - check()
     test("recursive_import_ok") - check()
     test("recursive_object") - check()
+    test("regex") - check()
     test("sanity") - checkGolden()
     test("sanity2") - checkGolden()
     test("shebang") - check()