Fix issue 17392 - Add Dub file for the lexer and parser

The Dub file is intended to make the the compiler available as a
library through Dub.

Author: jacob-carlborg

dub.sdl

@@ -0,0 +1,63 @@
+name "dmd"
+description "The DMD compiler"
+authors "Walter Bright"
+copyright "Copyright © 1999-2017, Digital Mars"
+license "BSL-1.0"
+
+targetType "library"
+
+// This is required because otherwise Dub will include all source files in src.
+// Just picked an existing directory which doesn't contain any D source files.
+sourcePaths "docs"
+
+// This is required because Dub needs to have at lease one source file
+sourceFiles "src/ddmd/package.d"
+
+dependency ":parser" version="*"
+
+subPackage {
+  name "idgen"
+  targetType "executable"
+  sourcePaths "docs"
+  sourceFiles "src/ddmd/idgen.d"
+}
+
+subPackage {
+  name "root"
+  targetType "library"
+  sourcePaths "src/ddmd/root"
+}
+
+subPackage {
+  name "lexer"
+  targetType "library"
+  sourcePaths "docs"
+
+  sourceFiles \
+    "src/ddmd/entity.d" \
+    "src/ddmd/errors.d" \
+    "src/ddmd/globals.d" \
+    "src/ddmd/id.d" \
+    "src/ddmd/identifier.d" \
+    "src/ddmd/lexer.d" \
+    "src/ddmd/tokens.d" \
+    "src/ddmd/utf.d"
+
+  // stringImportPaths cannot be used because it will make Dub extremely slow
+  dflags "-J$PACKAGE_DIR"
+
+  preBuildCommands "dub run dmd:idgen -- $PACKAGE_DIR"
+  dependency "dmd:root" version="*"
+}
+
+subPackage {
+  name "parser"
+  targetType "library"
+  sourcePaths "docs"
+
+  sourceFiles \
+    "src/ddmd/astnull.d" \
+    "src/ddmd/parse.d"
+
+  dependency "dmd:lexer" version="*"
+}

src/ddmd/idgen.d

@@ -372,17 +372,46 @@ Msgtable[] msgtable =
     { "udaSelector", "selector" },
 ];
 
+// Copied from std.string
+immutable(char)* toStringz(in string s) @trusted pure nothrow
+{
+    if (s.length == 0) return "".ptr;
+
+    immutable p = s.ptr + s.length;
+
+    if ((cast(size_t) p & 3) && *p == 0)
+        return &s[0];
+
+    auto copy = new char[s.length + 1];
+    copy[0 .. s.length] = s[];
+    copy[s.length] = 0;
 
-int main()
+    return &(cast(immutable(char)*)copy)[0];
+}
+
+int main(string[] args)
 {
+    // If an argument is given it's expected to be the full path of the project
+    // root directory with a trailing slash. The argument is used when building
+    // with Dub.
+    immutable workingDirectory = args.length == 2 ? args[1] ~ "src/" : "./";
+
+    FILE* openFile(string path)
     {
-        auto fp = fopen("ddmd/id.h","wb");
+        immutable fullPath = toStringz(workingDirectory ~ path);
+        auto fp = fopen(fullPath,"wb");
         if (!fp)
         {
-            printf("can't open ddmd/id.h\n");
+            printf("can't open %s\n", fullPath);
             exit(EXIT_FAILURE);
         }
 
+        return fp;
+    }
+
+    {
+        auto fp = openFile("ddmd/id.h");
+
         fprintf(fp, "// File generated by idgen.d\n");
         fprintf(fp, "#ifndef DMD_ID_H\n");
         fprintf(fp, "#define DMD_ID_H 1\n");
@@ -404,12 +433,7 @@ int main()
     }
 
     {
-        auto fp = fopen("ddmd/id.d","wb");
-        if (!fp)
-        {
-            printf("can't open ddmd.id.d\n");
-            exit(EXIT_FAILURE);
-        }
+        auto fp = openFile("ddmd/id.d");
 
         fprintf(fp, "// File generated by idgen.d\n");
         fprintf(fp, "\n");

src/ddmd/package.d

@@ -0,0 +1 @@
+module ddmd;

test/dub_package/.gitignore

@@ -0,0 +1,5 @@
+.dub
+docs.json
+__dummy.html
+*.o
+*.obj

test/dub_package/dub.sdl

@@ -0,0 +1,5 @@
+name "dmd-dub-test"
+description "Test of the DMD Dub package"
+license "BSL 1.0"
+
+dependency "dmd" path="../../"

test/dub_package/source/app.d

@@ -0,0 +1,42 @@
+void main()
+{
+}
+
+// lexer
+unittest
+{
+    import ddmd.lexer;
+    import ddmd.tokens;
+
+    immutable expected = [
+        TOKvoid,
+        TOKidentifier,
+        TOKlparen,
+        TOKrparen,
+        TOKlcurly,
+        TOKrcurly
+    ];
+
+    immutable sourceCode = "void test() {} // foobar";
+    scope lexer = new Lexer("test", sourceCode.ptr, 0, sourceCode.length, 0, 0);
+    lexer.nextToken;
+
+    TOK[] result;
+
+    do
+    {
+        result ~= lexer.token.value;
+    } while (lexer.nextToken != TOKeof);
+
+    assert(result == expected);
+}
+
+// parser
+unittest
+{
+    import ddmd.astnull;
+    import ddmd.parse;
+
+    scope parser = new Parser!ASTNull(null, null, false);
+    assert(parser !is null);
+}

travis.sh

@@ -54,6 +54,7 @@ rebuild() {
 
 # test druntime, phobos, dmd
 test() {
+    test_dub_package
     make -j$N -C ../druntime -f posix.mak MODEL=$MODEL unittest
     make -j$N -C ../phobos -f posix.mak MODEL=$MODEL unittest
     test_dmd
@@ -69,6 +70,13 @@ test_dmd() {
     fi
 }
 
+# test dub package
+test_dub_package() {
+    pushd test/dub_package
+    dub test
+    popd
+}
+
 for proj in druntime phobos; do
     if [ $TRAVIS_BRANCH != master ] && [ $TRAVIS_BRANCH != stable ] &&
            ! git ls-remote --exit-code --heads https://github.com/dlang/$proj.git $TRAVIS_BRANCH > /dev/null; then