From 3ba6e96e03c6740e99af8f48cf2cef6ee68b99e7 Mon Sep 17 00:00:00 2001 From: Stephen Amar Date: Sat, 18 Jan 2025 21:24:07 -0800 Subject: [PATCH] Add FileTests to JS --- build.sc | 45 ++++++++- sjsonnet/src-js/sjsonnet/SjsonnetMain.scala | 7 +- .../sjsonnet/CachedResolvedFile.scala | 6 +- sjsonnet/src/sjsonnet/Importer.scala | 11 +++ .../resources/test_suite/regex_js.jsonnet | 72 ++++++++++++++ .../test_suite/stdlib_native.jsonnet | 4 +- sjsonnet/test/src-js/sjsonnet/FileTests.scala | 97 +++++++++++++++++++ .../test/src-native/sjsonnet/FileTests.scala | 1 + 8 files changed, 236 insertions(+), 7 deletions(-) create mode 100644 sjsonnet/test/resources/test_suite/regex_js.jsonnet create mode 100644 sjsonnet/test/src-js/sjsonnet/FileTests.scala diff --git a/build.sc b/build.sc index 88457977..63c4125c 100644 --- a/build.sc +++ b/build.sc @@ -1,6 +1,8 @@ -import mill._, scalalib._, publish._, scalajslib._, scalanativelib._, scalanativelib.api._ +import mill._, scalalib._, publish._, scalajslib._, scalanativelib._, scalanativelib.api._, scalajslib.api._ import $ivy.`com.lihaoyi::mill-contrib-jmh:` import contrib.jmh.JmhModule +import java.util.Base64 +import java.nio.charset.StandardCharsets val sjsonnetVersion = "0.4.14" @@ -57,17 +59,58 @@ object sjsonnet extends Module { trait SjsonnetJsModule extends SjsonnetCrossModule with ScalaJSModule{ def millSourcePath = super.millSourcePath / os.up def scalaJSVersion = "1.17.0" + def esVersion = ESVersion.ES2018 def sources = T.sources( this.millSourcePath / "src", this.millSourcePath / "src-js", this.millSourcePath / "src-jvm-js" ) object test extends ScalaJSTests with CrossTests { + def jsEnvConfig = JsEnvConfig.NodeJs(args=List("--stack-size=" + 100 * 1024)) def sources = T.sources( this.millSourcePath / "src", this.millSourcePath / "src-js", this.millSourcePath / "src-jvm-js" ) + def generatedSources = T{ + val files = os.walk(this.millSourcePath / "resources").filterNot(os.isDir).map(p => p.relativeTo(this.millSourcePath / "resources") -> os.read.bytes(p)).toMap + os.write( + T.ctx().dest / "TestResources.scala", + s"""package sjsonnet + | + |object TestResources{ + | val files = Map( + |""".stripMargin) + for((k, v) <- files) { + val name = k.toString.replaceAll("/", "_").replaceAll("\\.", "_").replaceAll("-", "_") + val values = Base64.getEncoder().encodeToString(v).grouped(65535).toSeq + os.write( + T.ctx().dest / s"$name.scala", + s"""package sjsonnet + | + |import java.util.Base64 + | + |object $name { + | def contentArr = Seq( + | ${values.map("\"" + _ + "\"").mkString(",\n ")} + | ) + | def content = Base64.getDecoder().decode(contentArr.mkString) + |} + |""".stripMargin) + os.write.append( + T.ctx().dest / "TestResources.scala", + s""" "$k" -> $name.content, + |""".stripMargin + ) + } + os.write.append( + T.ctx().dest / "TestResources.scala", + s""" ) + |} + |""".stripMargin + ) + Seq(PathRef(T.ctx().dest / "TestResources.scala")) ++ files.keys.map(p => PathRef(T.ctx().dest / s"${p.toString.replaceAll("/", "_").replaceAll("\\.", "_").replaceAll("-", "_")}.scala")) + } } } diff --git a/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala b/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala index 51e5047a..77ff15cb 100644 --- a/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala +++ b/sjsonnet/src-js/sjsonnet/SjsonnetMain.scala @@ -12,7 +12,7 @@ object SjsonnetMain { tlaVars: js.Any, wd0: String, importResolver: js.Function2[String, String, String], - importLoader: js.Function1[String, String], + importLoader: js.Function2[String, Boolean, Either[String, Array[Byte]]], preserveOrder: Boolean = false): js.Any = { val interp = new Interpreter( ujson.WebJson.transform(extVars, ujson.Value).obj.toMap.map{case (k, ujson.Str(v)) => (k, v)}, @@ -25,7 +25,10 @@ object SjsonnetMain { case s => Some(JsVirtualPath(s)) } def read(path: Path, binaryData: Boolean): Option[ResolvedFile] = - Option(StaticResolvedFile(importLoader(path.asInstanceOf[JsVirtualPath].path))) + importLoader(path.asInstanceOf[JsVirtualPath].path, binaryData) match { + case Left(s) => Some(StaticResolvedFile(s)) + case Right(arr) => Some(StaticBinaryResolvedFile(arr)) + } }, parseCache = new DefaultParseCache, new Settings(preserveOrder = preserveOrder), diff --git a/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala b/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala index d8e2df2f..0fddc037 100644 --- a/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala +++ b/sjsonnet/src-jvm-native/sjsonnet/CachedResolvedFile.scala @@ -26,11 +26,13 @@ class CachedResolvedFile(val resolvedImportPath: OsPath, memoryLimitBytes: Long, // Assert that the file is less than limit assert(jFile.length() <= memoryLimitBytes, s"Resolved import path $resolvedImportPath is too large: ${jFile.length()} bytes > ${memoryLimitBytes} bytes") - private[this] val resolvedImportContent: StaticResolvedFile = { + private[this] val resolvedImportContent: ResolvedFile = { // TODO: Support caching binary data - if (jFile.length() > cacheThresholdBytes || binaryData) { + if (jFile.length() > cacheThresholdBytes) { // If the file is too large, then we will just read it from disk null + } else if (binaryData) { + StaticBinaryResolvedFile(readRawBytes(jFile)) } else { StaticResolvedFile(readString(jFile)) } diff --git a/sjsonnet/src/sjsonnet/Importer.scala b/sjsonnet/src/sjsonnet/Importer.scala index 7974b761..c6f3700d 100644 --- a/sjsonnet/src/sjsonnet/Importer.scala +++ b/sjsonnet/src/sjsonnet/Importer.scala @@ -164,6 +164,17 @@ case class StaticResolvedFile(content: String) extends ResolvedFile { override def readRawBytes(): Array[Byte] = content.getBytes(StandardCharsets.UTF_8) } +case class StaticBinaryResolvedFile(content: Array[Byte]) extends ResolvedFile { + def getParserInput(): ParserInput = ??? // Not used for binary imports + + def readString(): String = ??? // Not used for binary imports + + // We just cheat, the content hash can be the content itself for static imports + lazy val contentHash: String = content.hashCode().toString + + override def readRawBytes(): Array[Byte] = content +} + class CachedImporter(parent: Importer) extends Importer { val cache = mutable.HashMap.empty[Path, ResolvedFile] diff --git a/sjsonnet/test/resources/test_suite/regex_js.jsonnet b/sjsonnet/test/resources/test_suite/regex_js.jsonnet new file mode 100644 index 00000000..16150a25 --- /dev/null +++ b/sjsonnet/test/resources/test_suite/regex_js.jsonnet @@ -0,0 +1,72 @@ +std.assertEqual(std.native('regexFullMatch')(@'e', 'hello'), null) && + +std.assertEqual( + std.native('regexFullMatch')(@'h.*o', 'hello'), + { + string: 'hello', + captures: [], + namedCaptures: {}, + } +) && + +std.assertEqual( + std.native('regexFullMatch')(@'h(.*)o', 'hello'), + { + string: 'hello', + captures: ['ell'], + namedCaptures: {}, + } +) && + +std.assertEqual( + std.native('regexFullMatch')(@'h(?P.*)o', 'hello'), + { + string: 'hello', + captures: ['ell'], + namedCaptures: { + mid: 'ell', + }, + } +) && + +std.assertEqual(std.native('regexPartialMatch')(@'world', 'hello'), null) && + +std.assertEqual( + std.native('regexPartialMatch')(@'e', 'hello'), + { + string: 'hello', + captures: [], + namedCaptures: {}, + } +) && + +std.assertEqual( + std.native('regexPartialMatch')(@'e(.*)o', 'hello'), + { + string: 'hello', + captures: ['ll'], + namedCaptures: {}, + } +) && + +std.assertEqual( + std.native('regexPartialMatch')(@'e(?P.*)o', 'hello'), + { + string: 'hello', + captures: ['ll'], + namedCaptures: { + mid: 'll', + }, + } +) && + +std.assertEqual(std.native('regexQuoteMeta')(@'1.5-2.0?'), '\\Q1.5-2.0?\\E') && + + +std.assertEqual(std.native('regexReplace')('wishyfishyisishy', @'ish', 'and'), 'wandyfishyisishy') && +std.assertEqual(std.native('regexReplace')('yabba dabba doo', @'b+', 'd'), 'yada dabba doo') && + +std.assertEqual(std.native('regexGlobalReplace')('wishyfishyisishy', @'ish', 'and'), 'wandyfandyisandy') && +std.assertEqual(std.native('regexGlobalReplace')('yabba dabba doo', @'b+', 'd'), 'yada dada doo') && + +true \ No newline at end of file diff --git a/sjsonnet/test/resources/test_suite/stdlib_native.jsonnet b/sjsonnet/test/resources/test_suite/stdlib_native.jsonnet index b7938da5..cd5f68be 100644 --- a/sjsonnet/test/resources/test_suite/stdlib_native.jsonnet +++ b/sjsonnet/test/resources/test_suite/stdlib_native.jsonnet @@ -483,8 +483,8 @@ std.assertEqual(std.setMember('a', []), false) && std.assertEqual(std.setMember('a', ['b', 'c']), false) && ( - if std.thisFile == '' then - // This happens when testing the unparser. + if std.thisFile == '' || std.thisFile == "(memory)" then + // This happens when testing the unparser or scala.js true else std.assertEqual(std.thisFile, 'stdlib_native.jsonnet') diff --git a/sjsonnet/test/src-js/sjsonnet/FileTests.scala b/sjsonnet/test/src-js/sjsonnet/FileTests.scala new file mode 100644 index 00000000..d14b9504 --- /dev/null +++ b/sjsonnet/test/src-js/sjsonnet/FileTests.scala @@ -0,0 +1,97 @@ +package sjsonnet + +import java.util.Base64 +import java.nio.charset.StandardCharsets +import scala.scalajs.js +import utest._ + +object FileTests extends TestSuite { + def joinPath(a: String, b: String) = { + val aStripped = if (a.endsWith("/")) a.substring(0, a.length - 1) else a + val bStripped = if (b.startsWith("/")) b.substring(1) else b + if (aStripped.isEmpty) + bStripped + else if (bStripped.isEmpty) + aStripped + else + aStripped + "/" + bStripped + } + + def eval(fileName: String) = { + SjsonnetMain.interpret( + new String(TestResources.files(joinPath("test_suite", fileName)), StandardCharsets.UTF_8), + js.Dictionary("var1" -> """"test"""", "var2" -> """local f(a, b) = {[a]: b, "y": 2}; f("x", 1)"""), + js.Dictionary("var1" -> """"test"""", "var2" -> """{"x": 1, "y": 2}"""), + "test_suite", + (wd: String, path: String) => joinPath(wd, path), + (path: String, binaryData: Boolean) => if (binaryData) { + Right(TestResources.files(joinPath("test_suite", path))) + } else { + Left(new String(TestResources.files(joinPath("test_suite", path)), StandardCharsets.UTF_8)) + } + ) + } + def check(expected: ujson.Value = ujson.True)(implicit tp: utest.framework.TestPath) = { + val res = ujson.WebJson.transform(eval(s"${tp.value.last}.jsonnet"), ujson.Value) + assert(res == expected) + res + } + def checkFail(expected: String)(implicit tp: utest.framework.TestPath) = { + try { + eval(s"test_suite/${tp.value.last}.jsonnet").asInstanceOf[String] + assert(false) + } catch { + case e: js.JavaScriptException => + assert(e.getMessage == expected) + } + } + def checkGolden()(implicit tp: utest.framework.TestPath) = { + check(ujson.read(new String(TestResources.files(joinPath("test_suite", s"${tp.value.last}.jsonnet.golden")), StandardCharsets.UTF_8))) + } + + def tests = Tests{ + test("arith_bool") - check() + test("arith_float") - check() + test("arith_string") - check() + test("array") - check() + test("assert") - check() + test("binary") - check() + test("comments") - check() + test("condition") - check() + // test("dos_line_endings") - checkGolden() + test("format") - check() + // test("formatter") - checkGolden() + test("formatting_braces") - checkGolden() + test("formatting_braces2") - checkGolden() + test("functions") - check() + test("import") - check() + test("invariant") - check() + test("invariant_manifest") - checkGolden() + test("local") - check() + test("merge") - check() + test("null") - check() + test("object") - check() + test("oop") - check() + test("oop_extra") - check() + test("parsing_edge_cases") - check() + test("precedence") - check() + test("recursive_function_native") - check() + test("recursive_import_ok") - check() + test("recursive_object") - check() + test("regex_js") - check() + test("sanity") - checkGolden() + test("sanity2") - checkGolden() + test("shebang") - check() + test("slice.sugar") - check() + test("std_all_hidden") - check() + test("stdlib_native") - check() + test("text_block") - check() + test("tla.simple")- check() + test("unicode") - check() + test("unix_line_endings") - checkGolden() + test("unparse") - checkGolden() + test("verbatim_strings") - check() + test("issue_127") - check() + } +} + diff --git a/sjsonnet/test/src-native/sjsonnet/FileTests.scala b/sjsonnet/test/src-native/sjsonnet/FileTests.scala index bd4ff5de..091a4bc2 100644 --- a/sjsonnet/test/src-native/sjsonnet/FileTests.scala +++ b/sjsonnet/test/src-native/sjsonnet/FileTests.scala @@ -56,6 +56,7 @@ object FileTests extends TestSuite{ test("recursive_function_native") - check() test("recursive_import_ok") - check() test("recursive_object") - check() + test("regex") - check() test("sanity") - checkGolden() test("sanity2") - checkGolden() test("shebang") - check()