From ca5384dd1b949ccc0e165831af0aceec024e2c5c Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Sat, 21 Jul 2018 16:26:09 +0200 Subject: [PATCH 1/2] Support parsing only specific unittests (e.g. @betterc-test) --- posix.mak | 6 ++- test/tests_extractor/attributes.d | 44 ++++++++++++++++++ test/tests_extractor/attributes.d.ext | 32 +++++++++++++ tests_extractor.d | 66 ++++++++++++++++++++++----- 4 files changed, 135 insertions(+), 13 deletions(-) create mode 100644 test/tests_extractor/attributes.d create mode 100644 test/tests_extractor/attributes.d.ext diff --git a/posix.mak b/posix.mak index d4d6051a85..c27c2926d4 100644 --- a/posix.mak +++ b/posix.mak @@ -107,8 +107,10 @@ $(ROOT)/tests_extractor: tests_extractor.d ################################################################################ test_tests_extractor: $(ROOT)/tests_extractor - $< -i ./test/tests_extractor/ascii.d | diff - ./test/tests_extractor/ascii.d.ext - $< -i ./test/tests_extractor/iteration.d | diff - ./test/tests_extractor/iteration.d.ext + for file in ascii iteration ; do \ + $< -i "./test/tests_extractor/$${file}.d" | diff -p - "./test/tests_extractor/$${file}.d.ext"; \ + done + $< -a betterc -i "./test/tests_extractor/attributes.d" | diff -p - "./test/tests_extractor/attributes.d.ext"; RDMD_TEST_COMPILERS = $(DMD) RDMD_TEST_EXECUTABLE = $(ROOT)/rdmd diff --git a/test/tests_extractor/attributes.d b/test/tests_extractor/attributes.d new file mode 100644 index 0000000000..1d7416a650 --- /dev/null +++ b/test/tests_extractor/attributes.d @@ -0,0 +1,44 @@ +module attributes; + +enum betterc; + +@betterc @safe @("foo") unittest +{ + assert(1 == 1); +} + +@safe @("foo") unittest +{ + assert(2 == 2); +} + +/// +@("foo") unittest +{ + assert(3 == 3); +} + +@("foo") @betterc unittest +{ + assert(4 == 4); +} + +@("betterc") @([1, 2, 3]) unittest +{ + assert(5 == 5); +} + +@nogc @("foo", "betterc", "bar") @safe unittest +{ + assert(6 == 6); +} + +@nogc @("foo", "better", "bar") @safe unittest +{ + assert(7 == 7); +} + +@("betterd") unittest +{ + assert(8 == 8); +} diff --git a/test/tests_extractor/attributes.d.ext b/test/tests_extractor/attributes.d.ext new file mode 100644 index 0000000000..cf8af609f7 --- /dev/null +++ b/test/tests_extractor/attributes.d.ext @@ -0,0 +1,32 @@ +# line 3 +unittest +{ + import attributes; + + assert(1 == 1); +} + +# line 19 +unittest +{ + import attributes; + + assert(4 == 4); +} + +# line 24 +unittest +{ + import attributes; + + assert(5 == 5); +} + +# line 29 +unittest +{ + import attributes; + + assert(6 == 6); +} + diff --git a/tests_extractor.d b/tests_extractor.d index f7099d21d6..de9c1387c9 100755 --- a/tests_extractor.d +++ b/tests_extractor.d @@ -7,7 +7,7 @@ dependency "libdparse" version="~>0.8.0" * Parses all public unittests that are visible on dlang.org * (= annotated with three slashes) * - * Copyright (C) 2017 by D Language Foundation + * Copyright (C) 2018 by D Language Foundation * * Author: Sebastian Wilzbach * @@ -33,11 +33,13 @@ class TestVisitor : ASTVisitor File outFile; ubyte[] sourceCode; string moduleName; + string[] attributes; - this(File outFile, ubyte[] sourceCode) + this(File outFile, ubyte[] sourceCode, string[] attributes) { this.outFile = outFile; this.sourceCode = sourceCode; + this.attributes = attributes; } alias visit = ASTVisitor.visit; @@ -58,13 +60,51 @@ class TestVisitor : ASTVisitor override void visit(const Declaration decl) { - if (decl.unittest_ !is null && hasDdocHeader(sourceCode, decl)) + if (decl.unittest_ !is null && shouldIncludeUnittest(decl)) print(decl.unittest_); decl.accept(this); } private: + + bool shouldIncludeUnittest(const Declaration decl) + { + if (!attributes.empty) + return filterForUDAs(decl); + else + return hasDdocHeader(sourceCode, decl); + } + + bool filterForUDAs(const Declaration decl) + { + foreach (attr; decl.attributes) + { + // check for @myArg + if (attributes.canFind(attr.atAttribute.identifier.text)) + return true; + + // support @("myArg") too + if (auto argList = attr.atAttribute.argumentList) + { + foreach (arg; argList.items) + { + if (auto unaryExp = cast(UnaryExpression) arg) + if (auto primaryExp = unaryExp.primaryExpression) + { + auto attribute = primaryExp.primary.text; + if (attribute.length >= 2) + { + attribute = attribute[1 .. $ - 1]; + if (attributes.canFind(attribute)) + return true; + } + } + } + } + } + return false; + } void print(const Unittest u) { /* @@ -93,7 +133,7 @@ private: } } -void parseFile(File inFile, File outFile) +void parseFile(File inFile, File outFile, string[] attributes) { import dparse.lexer; import dparse.parser : parseModule; @@ -111,11 +151,11 @@ void parseFile(File inFile, File outFile) RollbackAllocator rba; auto m = parseModule(tokens.array, inFile.name, &rba); - auto visitor = new TestVisitor(outFile, sourceCode); + auto visitor = new TestVisitor(outFile, sourceCode, attributes); visitor.visit(m); } -void parseFileDir(string inputDir, string fileName, string outputDir) +void parseFileDir(string inputDir, string fileName, string outputDir, string[] attributes) { import std.path : buildPath, dirSeparator, buildNormalizedPath; @@ -132,7 +172,7 @@ void parseFileDir(string inputDir, string fileName, string outputDir) // convert the file path to a nice output file, e.g. std/uni.d -> std_uni.d string outName = fileNameNormalized.replace(dirSeparator, "_"); - parseFile(File(fileName), File(buildPath(outputDir, outName), "w")); + parseFile(File(fileName), File(buildPath(outputDir, outName), "w"), attributes); } void main(string[] args) @@ -143,12 +183,15 @@ void main(string[] args) string inputDir; string outputDir = "./out"; string ignoredFilesStr; - string modulePrefix = ""; + string modulePrefix; + string attributesStr; auto helpInfo = getopt(args, config.required, "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir, "outputdir|o", "Folder to which the extracted test files should be saved (stdout for a single file)", &outputDir, - "ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr); + "ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr, + "attributes|a", "Comma-separated list of UDAs that the unittest should have", &attributesStr, + ); if (helpInfo.helpWanted) { @@ -162,6 +205,7 @@ to in the output directory. inputDir = inputDir.asNormalizedPath.array; Algebraic!(string, File) outputLocation = cast(string) outputDir.asNormalizedPath.array; + auto attributes = attributesStr.split(","); if (!exists(outputDir)) mkdir(outputDir); @@ -196,8 +240,8 @@ to in the output directory. { stderr.writeln("parsing ", file); outputLocation.visit!( - (string outputFolder) => parseFileDir(inputDir, file, outputFolder), - (File outputFile) => parseFile(File(file.name, "r"), outputFile), + (string outputFolder) => parseFileDir(inputDir, file, outputFolder, attributes), + (File outputFile) => parseFile(File(file.name, "r"), outputFile, attributes), ); } else From b2cb7b86d6ef2b1fe3ecb8bdc203c65ccf528337 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Tue, 31 Jul 2018 16:37:42 +0200 Subject: [PATCH 2/2] Add -betterC unittest output --- posix.mak | 1 + test/tests_extractor/betterc.d | 16 +++++++++++ test/tests_extractor/betterc.d.ext | 24 ++++++++++++++++ tests_extractor.d | 46 +++++++++++++++++++++--------- 4 files changed, 74 insertions(+), 13 deletions(-) create mode 100644 test/tests_extractor/betterc.d create mode 100644 test/tests_extractor/betterc.d.ext diff --git a/posix.mak b/posix.mak index c27c2926d4..1055e3c766 100644 --- a/posix.mak +++ b/posix.mak @@ -111,6 +111,7 @@ test_tests_extractor: $(ROOT)/tests_extractor $< -i "./test/tests_extractor/$${file}.d" | diff -p - "./test/tests_extractor/$${file}.d.ext"; \ done $< -a betterc -i "./test/tests_extractor/attributes.d" | diff -p - "./test/tests_extractor/attributes.d.ext"; + $< --betterC -i "./test/tests_extractor/betterc.d" | diff -p - "./test/tests_extractor/betterc.d.ext"; RDMD_TEST_COMPILERS = $(DMD) RDMD_TEST_EXECUTABLE = $(ROOT)/rdmd diff --git a/test/tests_extractor/betterc.d b/test/tests_extractor/betterc.d new file mode 100644 index 0000000000..54d9566889 --- /dev/null +++ b/test/tests_extractor/betterc.d @@ -0,0 +1,16 @@ +module betterc; + +/// +unittest +{ + int a = 1; + assert(a == 2); +} + +/// +unittest +{ + int b = 2; + assert(b == 2); + assert(b == 3); +} diff --git a/test/tests_extractor/betterc.d.ext b/test/tests_extractor/betterc.d.ext new file mode 100644 index 0000000000..083858a393 --- /dev/null +++ b/test/tests_extractor/betterc.d.ext @@ -0,0 +1,24 @@ +# line 2 +unittest +{ + import betterc; + + int a = 1; + assert(a == 2); +} + +# line 9 +unittest +{ + import betterc; + + int b = 2; + assert(b == 2); + assert(b == 3); +} + +extern(C) void main() +{ + static foreach(u; __traits(getUnitTests, __traits(parent, main))) + u(); +} diff --git a/tests_extractor.d b/tests_extractor.d index de9c1387c9..c7cf8e38d6 100755 --- a/tests_extractor.d +++ b/tests_extractor.d @@ -33,13 +33,13 @@ class TestVisitor : ASTVisitor File outFile; ubyte[] sourceCode; string moduleName; - string[] attributes; + VisitorConfig config; - this(File outFile, ubyte[] sourceCode, string[] attributes) + this(File outFile, ubyte[] sourceCode, VisitorConfig config) { this.outFile = outFile; this.sourceCode = sourceCode; - this.attributes = attributes; + this.config = config; } alias visit = ASTVisitor.visit; @@ -56,6 +56,15 @@ class TestVisitor : ASTVisitor moduleName = outFile.name.replace(".d", "").replace(dirSeparator, ".").replace(".package", ""); } m.accept(this); + // -betterC doesn't run unittests out of the box + if (config.betterCOutput) + { + outFile.writeln(q{extern(C) void main() +{ + static foreach(u; __traits(getUnitTests, __traits(parent, main))) + u(); +}}); + } } override void visit(const Declaration decl) @@ -70,7 +79,7 @@ private: bool shouldIncludeUnittest(const Declaration decl) { - if (!attributes.empty) + if (!config.attributes.empty) return filterForUDAs(decl); else return hasDdocHeader(sourceCode, decl); @@ -80,8 +89,11 @@ private: { foreach (attr; decl.attributes) { + if (attr.atAttribute is null) + continue; + // check for @myArg - if (attributes.canFind(attr.atAttribute.identifier.text)) + if (config.attributes.canFind(attr.atAttribute.identifier.text)) return true; // support @("myArg") too @@ -96,7 +108,7 @@ private: if (attribute.length >= 2) { attribute = attribute[1 .. $ - 1]; - if (attributes.canFind(attribute)) + if (config.attributes.canFind(attribute)) return true; } } @@ -133,7 +145,7 @@ private: } } -void parseFile(File inFile, File outFile, string[] attributes) +void parseFile(File inFile, File outFile, VisitorConfig visitorConfig) { import dparse.lexer; import dparse.parser : parseModule; @@ -151,11 +163,11 @@ void parseFile(File inFile, File outFile, string[] attributes) RollbackAllocator rba; auto m = parseModule(tokens.array, inFile.name, &rba); - auto visitor = new TestVisitor(outFile, sourceCode, attributes); + auto visitor = new TestVisitor(outFile, sourceCode, visitorConfig); visitor.visit(m); } -void parseFileDir(string inputDir, string fileName, string outputDir, string[] attributes) +void parseFileDir(string inputDir, string fileName, string outputDir, VisitorConfig visitorConfig) { import std.path : buildPath, dirSeparator, buildNormalizedPath; @@ -172,7 +184,13 @@ void parseFileDir(string inputDir, string fileName, string outputDir, string[] a // convert the file path to a nice output file, e.g. std/uni.d -> std_uni.d string outName = fileNameNormalized.replace(dirSeparator, "_"); - parseFile(File(fileName), File(buildPath(outputDir, outName), "w"), attributes); + parseFile(File(fileName), File(buildPath(outputDir, outName), "w"), visitorConfig); +} + +struct VisitorConfig +{ + string[] attributes; /// List of attributes to extract; + bool betterCOutput; /// Add custom extern(C) main method for running D's unittests } void main(string[] args) @@ -185,12 +203,14 @@ void main(string[] args) string ignoredFilesStr; string modulePrefix; string attributesStr; + VisitorConfig visitorConfig; auto helpInfo = getopt(args, config.required, "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir, "outputdir|o", "Folder to which the extracted test files should be saved (stdout for a single file)", &outputDir, "ignore", "Comma-separated list of files to exclude (partial matching is supported)", &ignoredFilesStr, "attributes|a", "Comma-separated list of UDAs that the unittest should have", &attributesStr, + "betterC", "Add custom extern(C) main method for running D's unittests", &visitorConfig.betterCOutput, ); if (helpInfo.helpWanted) @@ -205,7 +225,7 @@ to in the output directory. inputDir = inputDir.asNormalizedPath.array; Algebraic!(string, File) outputLocation = cast(string) outputDir.asNormalizedPath.array; - auto attributes = attributesStr.split(","); + visitorConfig.attributes = attributesStr.split(","); if (!exists(outputDir)) mkdir(outputDir); @@ -240,8 +260,8 @@ to in the output directory. { stderr.writeln("parsing ", file); outputLocation.visit!( - (string outputFolder) => parseFileDir(inputDir, file, outputFolder, attributes), - (File outputFile) => parseFile(File(file.name, "r"), outputFile, attributes), + (string outputFolder) => parseFileDir(inputDir, file, outputFolder, visitorConfig), + (File outputFile) => parseFile(File(file.name, "r"), outputFile, visitorConfig), ); } else