Cleanup tests

.gitignore

@@ -27,9 +27,9 @@ Makefile.depend
 Makefile.depend.bak
 bazel-*
 
-libjsonnet_test_file
-libjsonnet_test_snippet
-libjsonnet_test_locale
+libjsonnet_file_test
+libjsonnet_native_callbacks_test
+libjsonnet_locale_test
 libjsonnet_test
 libjsonnet++_test
 unicode_test

CMakeLists.txt

@@ -317,12 +317,31 @@ if(BUILD_TESTS)
         set_target_properties(${TESTNAME} PROPERTIES FOLDER tests)
     endfunction()
 
+    function(jsonnet_add_raw_test TESTNAME JSONNET_LIB_TARGET)
+        add_executable(${TESTNAME} ${ARGN})
+        target_link_libraries(${TESTNAME} ${JSONNET_LIB_TARGET})
+        set_target_properties(${TESTNAME} PROPERTIES FOLDER tests)
+        add_test(NAME ${TESTNAME} COMMAND ${TESTNAME})
+    endfunction()
+
     jsonnet_add_gtest(test_core_unicode jsonnet_lib_for_binaries core/unicode_test.cpp)
     jsonnet_add_gtest(test_core_lexer jsonnet_lib_for_binaries core/lexer_test.cpp)
     jsonnet_add_gtest(test_core_parser jsonnet_lib_for_binaries core/parser_test.cpp)
     jsonnet_add_gtest(test_core_libjsonnet jsonnet_lib_for_binaries core/libjsonnet_test.cpp)
     jsonnet_add_gtest(test_cpp_libjsonnet jsonnet_cpp_lib_for_binaries cpp/libjsonnet++_test.cpp)
-    jsonnet_add_gtest(test_cpp_libjsonnet_locale jsonnet_cpp_lib_for_binaries cpp/libjsonnet_test_locale.cpp)
+    jsonnet_add_gtest(test_cpp_libjsonnet_locale jsonnet_cpp_lib_for_binaries cpp/libjsonnet_locale_test.cpp)
+
+    jsonnet_add_raw_test(test_core_libjsonnet_native_callbacks jsonnet_lib_for_binaries core/libjsonnet_native_callbacks_test.c)
+
+    # core/libjsonnet_file_test needs an input file
+    add_executable(test_libjsonnet_file core/libjsonnet_file_test.c)
+    target_link_libraries(test_libjsonnet_file jsonnet_lib_for_binaries)
+    set_target_properties(test_libjsonnet_file PROPERTIES FOLDER tests)
+    add_test(
+        NAME test_libjsonnet_file
+        COMMAND test_libjsonnet_file "test_suite/object.jsonnet"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
 
     add_subdirectory(test_suite)
 endif()  # if(BUILD_TESTS)
@@ -386,7 +405,7 @@ if(CMAKE_VERSION VERSION_GREATER "3.25")
                 jsonnetfmt
                 jsonnet-man
                 jsonnetfmt-man
-            PROPERTIES 
+            PROPERTIES
                 TYPE
                 C_STANDARD
                 CXX_STANDARD

Makefile

@@ -151,10 +151,10 @@ PUBLIC_HEADERS := \
 	include/libjsonnet++.h
 
 PLAIN_TEST_SRC := \
-	core/libjsonnet_test_file.c \
-	core/libjsonnet_test_snippet.c
+	core/libjsonnet_file_test.c \
+	core/libjsonnet_native_callbacks_test.c
 GTEST_TEST_SRC := \
-	cpp/libjsonnet_test_locale.cpp \
+	cpp/libjsonnet_locale_test.cpp \
 	cpp/libjsonnet++_test.cpp \
 	core/libjsonnet_test.cpp \
 	core/lexer_test.cpp \
@@ -257,7 +257,7 @@ core/%.jsonnet.h: stdlib/%.jsonnet .makebuild/stdlib/to_c_array
 	.makebuild/stdlib/to_c_array "$<" "$@"
 
 # Plain-C tests (link to libjsonnet.so); don't use GoogleTest (which is C++ only)
-libjsonnet_test_file libjsonnet_test_snippet: %: .makebuild/core/%.c.o libjsonnet.so
+libjsonnet_file_test libjsonnet_native_callbacks_test: %: .makebuild/core/%.c.o libjsonnet.so
 	$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(TEST_RPATH_FLAG)
 
 ###
@@ -267,7 +267,7 @@ libjsonnet_test_file libjsonnet_test_snippet: %: .makebuild/core/%.c.o libjsonne
 ifeq ($(GTEST_ENABLED),yes)
 
 # C++ lib tests (links to libjsonnet++.so)
-libjsonnet_test_locale libjsonnet++_test: %: .makebuild/cpp/%.cpp.o libjsonnet++.so
+libjsonnet_locale_test libjsonnet++_test: %: .makebuild/cpp/%.cpp.o libjsonnet++.so
 	$(CXX) $(CXXFLAGS) $(GTEST_CXXFLAGS) -o $@ $^ $(LDFLAGS) $(GTEST_LDFLAGS) $(TEST_RPATH_FLAG)
 
 # C++ tests of the C API (link to libjsonnet.so)

core/BUILD

@@ -83,3 +83,17 @@ cc_test(
         "@googletest//:gtest_main",
     ],
 )
+
+cc_test(
+    name = "libjsonnet_native_callbacks_test",
+    srcs = ["libjsonnet_native_callbacks_test.c"],
+    deps = [":libjsonnet"],
+)
+
+cc_test(
+    name = "libjsonnet_file_test",
+    srcs = ["libjsonnet_file_test.c"],
+    args = ["$(rootpath //test_suite:test_input_for_core_test)"],
+    data = ["//test_suite:test_input_for_core_test"],
+    deps = [":libjsonnet"],
+)

core/libjsonnet_test_snippet.c → core/libjsonnet_native_callbacks_test.c

@@ -84,31 +84,101 @@ static JJV *native_build(void *ctx, const JJV * const *argv, int *succ)
     return obj_top;
 }
 
-int main(int argc, const char **argv)
+#define BUILD_OUTPUT \
+    "{\"field\":[[\"Test 1.1\",\"Test 1.2\",\"Test 1.3\",true,42,null,{}],"\
+    "[\"Test 2.1\",\"Test 2.2\",\"Test 2.3\",false,-42,null,{\"f\":\"foo\",\"g\":\"bar\"}]]}"
+
+static int eval_and_check(const char *test_name, const char *input, const char *expect, int expect_error)
 {
-    int error;
-    char *output;
-    struct JsonnetVm *vm;
-    const char *params0[] = {NULL};
-    const char *params1[] = {"a", NULL};
-    const char *params2[] = {"a", "b", NULL};
-    if (argc != 2) {
-        fprintf(stderr, "libjsonnet_test_snippet <string>\n");
-        return EXIT_FAILURE;
+    struct JsonnetVm *vm = jsonnet_make();
+
+    static const char *PARAMS_0[] = {NULL};
+    static const char *PARAMS_1[] = {"a", NULL};
+    static const char *PARAMS_2[] = {"a", "b", NULL};
+    jsonnet_native_callback(vm, "build", native_build, vm, PARAMS_0);
+    jsonnet_native_callback(vm, "square", native_square, vm, PARAMS_1);
+    jsonnet_native_callback(vm, "concat", native_concat, vm, PARAMS_2);
+
+    int error = 0;
+    char *output = jsonnet_evaluate_snippet(vm, "snippet", input, &error);
+    if (expect && strcmp(output, expect)) {
+        error = 1;
+        fprintf(stderr, "FAIL: %s produced wrong output. Want:\n%s\n----- Got -----\n%s\n-----\n", test_name, expect, output);
+    } else {
+        if (error && expect_error) {
+            error = 0;
+        } else if (error) {
+            fprintf(stderr, "FAIL: %s failed evaluation: %s\n", test_name, output);
+        } else if (expect_error) {
+            fprintf(stderr, "FAIL: %s evaluated unexpectedly. output was:\n%s\n", test_name, output);
+        }
     }
-    vm = jsonnet_make();
-    jsonnet_native_callback(vm, "concat", native_concat, vm, params2);
-    jsonnet_native_callback(vm, "square", native_square, vm, params1);
-    jsonnet_native_callback(vm, "build", native_build, vm, params0);
-    output = jsonnet_evaluate_snippet(vm, "snippet", argv[1], &error);
-    if (error) {
-        fprintf(stderr, "%s", output);
-        jsonnet_realloc(vm, output, 0);
-        jsonnet_destroy(vm);
-        return EXIT_FAILURE;
-    } 
-    printf("%s", output);
     jsonnet_realloc(vm, output, 0);
     jsonnet_destroy(vm);
-    return EXIT_SUCCESS;
+    if (!error) {
+        fprintf(stderr, "PASS: %s\n", test_name);
+    }
+    return !error;
+}
+
+static int run_tests()
+{
+    int fail_count = 0;
+
+    fail_count += !eval_and_check("build",
+        "std.assertEqual(std.native('build')()," BUILD_OUTPUT ") && true",
+        "true\n", 0);
+
+    fail_count += !eval_and_check("square_noarg",
+        "std.native('square')()",
+        "RUNTIME ERROR: function parameter a not bound in call.\n\tsnippet:1:1-23\t\n", 1);
+
+    fail_count += !eval_and_check("square_badarg",
+        "std.native('square')('prune')",
+        "RUNTIME ERROR: Bad param 'a'.\n\tsnippet:1:1-30\t\n", 1);
+
+    fail_count += !eval_and_check("square_badarg2",
+        "std.native('square')({})",
+        "RUNTIME ERROR: native extensions can only take primitives.\n\tsnippet:1:1-25\t\n", 1);
+
+    fail_count += !eval_and_check("square",
+        "std.native('square')(42.5)",
+        "1806.25\n", 0);
+
+    fail_count += !eval_and_check("concat_noarg0",
+        "std.native('concat')()",
+        "RUNTIME ERROR: function parameter a not bound in call.\n\tsnippet:1:1-23\t\n", 1);
+
+    fail_count += !eval_and_check("concat_noarg1",
+        "std.native('concat')('hello')",
+        "RUNTIME ERROR: function parameter b not bound in call.\n\tsnippet:1:1-30\t\n", 1);
+
+    fail_count += !eval_and_check("concat_badarg1",
+        "std.native('concat')(false, 'hello')",
+        "RUNTIME ERROR: Bad params.\n\tsnippet:1:1-37\t\n", 1);
+
+    fail_count += !eval_and_check("concat_badarg2",
+        "std.native('concat')('hello', true)",
+        "RUNTIME ERROR: Bad params.\n\tsnippet:1:1-36\t\n", 1);
+
+    fail_count += !eval_and_check("concat_badarg3",
+        "std.native('concat')(99, 100)",
+        "RUNTIME ERROR: Bad params.\n\tsnippet:1:1-30\t\n", 1);
+
+    fail_count += !eval_and_check("concat",
+        "std.native('concat')('hello', 'world')",
+        "\"helloworld\"\n", 0);
+
+    return !fail_count;
+}
+
+int main(int argc, const char **argv)
+{
+    (void)argv; /* not unused */
+    if (argc > 1) {
+        fprintf(stderr, "libjsonnet_native_callbacks_test does not support any command line arguments");
+        return EXIT_FAILURE;
+    }
+    const int ok = run_tests();
+    return (ok ? EXIT_SUCCESS : EXIT_FAILURE);
 }

cpp/BUILD

@@ -25,7 +25,7 @@ cc_test(
 
 cc_test(
     name = "libjsonnet_locale_test",
-    srcs = ["libjsonnet_test_locale.cpp"],
+    srcs = ["libjsonnet_locale_test.cpp"],
     data = ["//cpp/testdata"],
     deps = [
         ":libjsonnet++",

tests.sh

@@ -3,14 +3,14 @@ set -e
 
 JSONNET_BIN="${JSONNET_BIN:-./jsonnet}"
 TEST_SNIPPET="std.assertEqual(({ x: 1, y: self.x } { x: 2 }).y, 2)"
-printf "snippet: "
+printf "snippet output (should be true): "
 "$JSONNET_BIN" -e "${TEST_SNIPPET}" || FAIL=TRUE
 
 if [ -z "$DISABLE_LIB_TESTS" ]; then
-    printf 'libjsonnet_test_snippet: '
-    ./libjsonnet_test_snippet "${TEST_SNIPPET}" || FAIL=TRUE
-    printf 'libjsonnet_test_file: '
-    ./libjsonnet_test_file "test_suite/object.jsonnet" || FAIL=TRUE
+    printf 'libjsonnet_native_callbacks_test: '
+    ./libjsonnet_native_callbacks_test || FAIL=TRUE
+    printf 'libjsonnet_file_test: '
+    ./libjsonnet_file_test "test_suite/object.jsonnet" || FAIL=TRUE
 
     for gtest_bin in unicode_test lexer_test parser_test libjsonnet_test libjsonnet++_test; do
         if [[ -x "./${gtest_bin}" ]]; then