Expose D frontend via DUB

dub.sdl

@@ -5,7 +5,7 @@ copyright "Copyright © 1999-2017, The D Language Foundation"
 license "BSL-1.0"
 
 targetType "none"
-dependency ":parser" version="*"
+dependency ":frontend" version="*"
 
 subPackage {
   name "root"
@@ -49,3 +49,20 @@ subPackage {
 
   dependency "dmd:lexer" version="*"
 }
+
+subPackage {
+  name "frontend"
+  targetType "library"
+  preGenerateCommands `cd "$${DUB_PACKAGE_DIR}" && ./config.sh generated/dub VERSION /etc`
+  stringImportPaths "generated/dub"
+  stringImportPaths "res"
+  versions "NoBackend"
+  versions "GC"
+  versions "NoMain"
+  sourcePaths "src/dmd"
+  excludedSourceFiles "src/dmd/backend/*"
+  excludedSourceFiles "src/dmd/{e2ir,eh,glue,iasm,objc_glue,s2ir,tocsym,toctype,toobj,todt,toir}.d"
+  excludedSourceFiles "src/dmd/{scan,lib}{mach,mscoff,omf}.d" platform="linux"
+  excludedSourceFiles "src/dmd/{scan,lib}{elf,mscoff,omf}.d" platform="osx"
+  excludedSourceFiles "src/dmd/{scan,lib}{elf,mach,mscoff}.d" platform="windows"
+}

src/dmd/frontend.d

@@ -0,0 +1,56 @@
+/**
+ * Compiler implementation of the
+ * $(LINK2 http://www.dlang.org, D programming language).
+ *
+ * This module contains high-level interfaces for interacting
+  with DMD as a library.
+ *
+ * Copyright:   Copyright (c) 1999-2017 by The D Language Foundation, All Rights Reserved
+ * Authors:     $(LINK2 http://www.digitalmars.com, Walter Bright)
+ * License:     $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
+ * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/id.d, _id.d)
+ */
+module dmd.frontend;
+
+// Online documentation: https://dlang.org/phobos/dmd_frontend.html
+
+/**
+Initializes the DMD compiler
+*/
+void initDMD()
+{
+    import dmd.dmodule : Module;
+    import dmd.globals : global;
+    import dmd.id : Id;
+    import dmd.mtype : Type;
+    import dmd.target : Target;
+    import dmd.expression : Expression;
+    import dmd.objc : Objc;
+    import dmd.builtin : builtin_init;
+
+    global._init;
+
+    version(linux)
+        global.params.isLinux = 1;
+    else version(OSX)
+        global.params.isOSX = 1;
+    else version(FreeBSD)
+        global.params.isFreeBSD = 1;
+    else version(Windows)
+        global.params.isWindows = 1;
+    else version(Solaris)
+        global.params.isSolaris = 1;
+    else version(OpenBSD)
+        global.params.isOpenBSD = 1;
+    else
+        static assert(0, "OS not supported yet.");
+
+    Type._init();
+    Id.initialize();
+    Module._init();
+    Target._init();
+    Expression._init();
+    Objc._init();
+    builtin_init();
+}
+

src/dmd/root/rmem.d

@@ -55,6 +55,11 @@ version (GC)
         }
     }
 
+    extern (C) void* allocmemory(size_t m_size) nothrow
+    {
+        return GC.malloc(m_size);
+    }
+
     extern (C++) const __gshared Mem mem;
 }
 else

test/dub_package/.gitignore

@@ -1,6 +1,13 @@
+# Ignore all binary files (files without an extension)
+# Start: Ignore everything
+*
+# Include directories
+!*/
+# Include all files with extensions
+!*.*
+
 .dub
 docs.json
 __dummy.html
 *.o
 *.obj
-/parser

test/dub_package/README.md

@@ -0,0 +1,21 @@
+DMD as a library
+================
+
+The tests in this module are high-level and mostly indented to make sure all
+necessary modules are included in the Dub package.
+
+The tests are executable single-file packages for the D's package manager `dub`
+and can be directly executed:
+
+```bash
+./lexer.d
+```
+
+If you want to see the log output or want to pass additional options, use `--single`:
+
+```bash
+dub --single -v lexer.d
+```
+
+If you don't have `dub` installed on your system,
+[install an official release](https://dlang.org/download.html).

test/dub_package/frontend.d

@@ -0,0 +1,162 @@
+#!/usr/bin/env dub
+/+dub.sdl:
+dependency "dmd" path="../.."
++/
+import std.stdio;
+
+// add import paths
+void addImports(T)(T path)
+{
+    import dmd.globals : global;
+    import dmd.arraytypes : Strings;
+
+    stderr.writefln("addImport: %s", path);
+
+    Strings* res = new Strings();
+    foreach (p; path)
+    {
+        import std.string : toStringz;
+        Strings* a = new Strings();
+        a.push(p.toStringz);
+        res.append(a);
+    }
+    global.path = res;
+}
+
+// finds a dmd.conf and parses it for import paths
+auto findImportPaths()
+{
+    import std.file : exists, getcwd;
+    import std.string : fromStringz, toStringz;
+    import std.path : buildPath, buildNormalizedPath, dirName;
+    import std.process : env = environment, execute;
+    import dmd.dinifile : findConfFile;
+    import dmd.errors : fatal;
+    import std.algorithm, std.range, std.regex;
+
+    auto dmdEnv = env.get("DMD", "dmd");
+    auto whichDMD = execute(["which", dmdEnv]);
+    if (whichDMD.status != 0)
+    {
+        stderr.writeln("Can't find DMD.");
+        fatal;
+    }
+
+    immutable dmdFilePath = whichDMD.output;
+    string iniFile;
+
+    if (dmdEnv.canFind("ldmd"))
+    {
+        immutable ldcConfig = "ldc2.conf";
+        immutable binDir = dmdFilePath.dirName;
+        // https://wiki.dlang.org/Using_LDC
+        auto ldcConfigs = [
+            getcwd.buildPath(ldcConfig),
+            binDir.buildPath(ldcConfig),
+            binDir.dirName.buildPath("etc", ldcConfig),
+            "~/.ldc".buildPath(ldcConfig),
+            binDir.buildPath("etc", ldcConfig),
+            binDir.buildPath("etc", "ldc", ldcConfig),
+            "/etc".buildPath(ldcConfig),
+            "/etc/ldc".buildPath(ldcConfig),
+        ].filter!exists;
+        assert(!ldcConfigs.empty, "No ldc2.conf found");
+        iniFile = ldcConfigs.front;
+    }
+    else
+    {
+        auto f = findConfFile(dmdFilePath.toStringz, "dmd.conf");
+        iniFile = f.fromStringz.idup;
+        assert(iniFile.exists, "No dmd.conf found.");
+    }
+
+    return File(iniFile, "r")
+        .byLineCopy
+        .map!(l => l.matchAll(`-I[^ "]+`.regex)
+                    .joiner
+                    .map!(a => a.drop(2)
+                                .replace("%@P%", dmdFilePath.dirName)
+                                .replace("%%ldcbinarypath%%", dmdFilePath.dirName)))
+        .joiner
+        .array
+        .sort
+        .uniq
+        .map!buildNormalizedPath;
+}
+
+// test frontend
+void main()
+{
+    import dmd.astcodegen : ASTCodegen;
+    import dmd.dmodule : Module;
+    import dmd.globals : global, Loc;
+    import dmd.frontend : initDMD;
+    import dmd.parse : Parser;
+    import dmd.statement : Identifier;
+    import dmd.tokens : TOKeof;
+    import dmd.id : Id;
+
+    initDMD;
+    findImportPaths.addImports;
+
+    auto parse(Module m, string code)
+    {
+        scope p = new Parser!ASTCodegen(m, code, false);
+        p.nextToken; // skip the initial token
+        auto members = p.parseModule;
+        assert(!p.errors, "Parsing error occurred.");
+        return members;
+    }
+
+    Identifier id = Identifier.idPool("test");
+    auto m = new Module("test.d", id, 0, 0);
+    m.members = parse(m, q{
+        void foo()
+        {
+            foreach (i; 0..10) {}
+        }
+    });
+
+    void semantic()
+    {
+        import dmd.dsymbolsem : dsymbolSemantic;
+        import dmd.semantic : semantic2, semantic3;
+
+        m.importedFrom = m;
+        m.importAll(null);
+        m.dsymbolSemantic(null);
+        m.semantic2(null);
+        m.semantic3(null);
+    }
+    semantic();
+
+    auto prettyPrint()
+    {
+        import dmd.root.outbuffer: OutBuffer;
+        import dmd.hdrgen : HdrGenState, PrettyPrintVisitor;
+
+        OutBuffer buf = { doindent: 1 };
+        HdrGenState hgs = { fullDump: 1 };
+        scope PrettyPrintVisitor ppv = new PrettyPrintVisitor(&buf, &hgs);
+        m.accept(ppv);
+        return buf;
+    }
+    auto buf = prettyPrint();
+
+    auto expected =q{import object;
+void foo()
+{
+    {
+        int __key3 = 0;
+        int __limit4 = 10;
+        for (; __key3 < __limit4; __key3 += 1)
+        {
+            int i = __key3;
+        }
+    }
+}
+};
+    import std.string : replace, fromStringz;
+    auto generated = buf.extractData.fromStringz.replace("\t", "    ");
+    assert(expected == generated, generated);
+}

test/dub_package/lexer.d

@@ -0,0 +1,31 @@
+#!/usr/bin/env dub
+/+dub.sdl:
+dependency "dmd" path="../.."
++/
+void main()
+{
+    import dmd.lexer;
+    import dmd.tokens;
+
+    immutable expected = [
+        TOKvoid,
+        TOKidentifier,
+        TOKlparen,
+        TOKrparen,
+        TOKlcurly,
+        TOKrcurly
+    ];
+
+    immutable sourceCode = "void test() {} // foobar";
+    scope lexer = new Lexer("test", sourceCode.ptr, 0, sourceCode.length, 0, 0);
+    lexer.nextToken;
+
+    TOK[] result;
+
+    do
+    {
+        result ~= lexer.token.value;
+    } while (lexer.nextToken != TOKeof);
+
+    assert(result == expected);
+}

test/dub_package/parser.d

@@ -1,44 +1,8 @@
+#!/usr/bin/env dub
 /+dub.sdl:
 dependency "dmd" path="../.."
 +/
-// The tests in this module are highlevel and mostly indented to make sure all
-// necessary modules are included in the Dub package.
-
 void main()
-{
-}
-
-// lexer
-unittest
-{
-    import dmd.lexer;
-    import dmd.tokens;
-
-    immutable expected = [
-        TOKvoid,
-        TOKidentifier,
-        TOKlparen,
-        TOKrparen,
-        TOKlcurly,
-        TOKrcurly
-    ];
-
-    immutable sourceCode = "void test() {} // foobar";
-    scope lexer = new Lexer("test", sourceCode.ptr, 0, sourceCode.length, 0, 0);
-    lexer.nextToken;
-
-    TOK[] result;
-
-    do
-    {
-        result ~= lexer.token.value;
-    } while (lexer.nextToken != TOKeof);
-
-    assert(result == expected);
-}
-
-// parser
-unittest
 {
     import dmd.astbase;
     import dmd.parse;

travis.sh

@@ -85,9 +85,16 @@ test_dmd() {
 # test dub package
 test_dub_package() {
     source ~/dlang/*/activate # activate host compiler
-    pushd test/dub_package
-    dub --single --build=unittest parser.d
-    popd
+    # GDC's standard library is too old for some example scripts
+    if [ "${DMD:-dmd}" == "gdmd" ] ; then
+        echo "Skipping DUB examples on GDC."
+    else
+        pushd test/dub_package
+        for file in *.d ; do
+            dub --single "$file"
+        done
+        popd
+    fi
     deactivate
 }