From cbd0d2788285390cd63776396ae004eb76ea968f Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Tue, 12 Dec 2017 04:10:26 +0100 Subject: [PATCH 1/7] Add allocmemory to Mem (GC-version) --- src/dmd/root/rmem.d | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/dmd/root/rmem.d b/src/dmd/root/rmem.d index a894b4cfe194..e8d01955cc1c 100644 --- a/src/dmd/root/rmem.d +++ b/src/dmd/root/rmem.d @@ -55,6 +55,11 @@ version (GC) } } + extern (C) void* allocmemory(size_t m_size) nothrow + { + return GC.malloc(m_size); + } + extern (C++) const __gshared Mem mem; } else From b203430b1b8367c20023cefa6ef497fd44acac64 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Mon, 11 Dec 2017 10:29:38 +0100 Subject: [PATCH 2/7] Expose D frontend via DUB --- dub.sdl | 103 ++++++++++++++++++++++++++++++++++++- test/dub_package/README.md | 21 ++++++++ test/dub_package/lexer.d | 31 +++++++++++ test/dub_package/parser.d | 38 +------------- 4 files changed, 155 insertions(+), 38 deletions(-) create mode 100644 test/dub_package/README.md create mode 100755 test/dub_package/lexer.d mode change 100644 => 100755 test/dub_package/parser.d diff --git a/dub.sdl b/dub.sdl index feeb2637287d..2eac444644b5 100644 --- a/dub.sdl +++ b/dub.sdl @@ -5,7 +5,7 @@ copyright "Copyright © 1999-2017, The D Language Foundation" license "BSL-1.0" targetType "none" -dependency ":parser" version="*" +dependency ":frontend" version="*" subPackage { name "root" @@ -49,3 +49,104 @@ subPackage { dependency "dmd:lexer" version="*" } + +subPackage { + name "frontend" + targetType "library" + stringImportPaths "res" + versions "NoBackend" + versions "GC" + versions "NoMain" + sourcePaths + + sourceFiles \ + "src/dmd/access.d" \ + "src/dmd/aggregate.d" \ + "src/dmd/aliasthis.d" \ + "src/dmd/apply.d" \ + "src/dmd/argtypes.d" \ + "src/dmd/arrayop.d" \ + "src/dmd/arraytypes.d" \ + "src/dmd/astcodegen.d" \ + "src/dmd/attrib.d" \ + "src/dmd/blockexit.d" \ + "src/dmd/builtin.d" \ + "src/dmd/canthrow.d" \ + "src/dmd/clone.d" \ + "src/dmd/complex.d" \ + "src/dmd/cond.d" \ + "src/dmd/constfold.d" \ + "src/dmd/cppmangle.d" \ + "src/dmd/cppmanglewin.d" \ + "src/dmd/ctfeexpr.d" \ + "src/dmd/dcast.d" \ + "src/dmd/dclass.d" \ + "src/dmd/declaration.d" \ + "src/dmd/delegatize.d" \ + "src/dmd/denum.d" \ + "src/dmd/dimport.d" \ + "src/dmd/dinifile.d" \ + "src/dmd/dinterpret.d" \ + "src/dmd/dmacro.d" \ + "src/dmd/dmangle.d" \ + "src/dmd/dmodule.d" \ + "src/dmd/doc.d" \ + "src/dmd/dscope.d" \ + "src/dmd/dstruct.d" \ + "src/dmd/dsymbol.d" \ + "src/dmd/dsymbolsem.d" \ + "src/dmd/dtemplate.d" \ + "src/dmd/dversion.d" \ + "src/dmd/escape.d" \ + "src/dmd/expression.d" \ + "src/dmd/expressionsem.d" \ + "src/dmd/func.d" \ + "src/dmd/gluelayer.d" \ + "src/dmd/hdrgen.d" \ + "src/dmd/id.d" \ + "src/dmd/impcnvtab.d" \ + "src/dmd/imphint.d" \ + "src/dmd/init.d" \ + "src/dmd/initsem.d" \ + "src/dmd/inline.d" \ + "src/dmd/inlinecost.d" \ + "src/dmd/intrange.d" \ + "src/dmd/json.d" \ + "src/dmd/lib.d" \ + "src/dmd/link.d" \ + "src/dmd/mars.d" \ + "src/dmd/mtype.d" \ + "src/dmd/nogc.d" \ + "src/dmd/nspace.d" \ + "src/dmd/objc.d" \ + "src/dmd/opover.d" \ + "src/dmd/optimize.d" \ + "src/dmd/parse.d" \ + "src/dmd/parsetimevisitor.d" \ + "src/dmd/printast.d" \ + "src/dmd/safe.d" \ + "src/dmd/sapply.d" \ + "src/dmd/semantic.d" \ + "src/dmd/sideeffect.d" \ + "src/dmd/statement.d" \ + "src/dmd/statement_rewrite_walker.d" \ + "src/dmd/statementsem.d" \ + "src/dmd/staticassert.d" \ + "src/dmd/staticcond.d" \ + "src/dmd/target.d" \ + "src/dmd/templateparamsem.d" \ + "src/dmd/traits.d" \ + "src/dmd/typesem.d" \ + "src/dmd/typinf.d" \ + "src/dmd/utils.d" \ + "src/dmd/visitor.d" + + sourceFiles "src/dmd/scanelf.d" \ + "src/dmd/libelf.d" platform="linux" + + sourceFiles "src/dmd/scanmach.d" \ + "src/dmd/libmach.d" platform="osx" + + dependency "dmd:parser" version="*" + dependency "dmd:lexer" version="*" +} diff --git a/test/dub_package/README.md b/test/dub_package/README.md new file mode 100644 index 000000000000..e885fd33d418 --- /dev/null +++ b/test/dub_package/README.md @@ -0,0 +1,21 @@ +DMD as a library +================ + +The tests in this module are high-level and mostly indented to make sure all +necessary modules are included in the Dub package. + +The tests are executable single-file packages for the D's package manager `dub` +and can be directly executed: + +```bash +./lexer.d +``` + +If you want to see the log output or want to pass additional options, use `--single`: + +```bash +dub --single -v lexer.d +``` + +If you don't have `dub` installed on your system, +[install an official release](https://dlang.org/download.html). diff --git a/test/dub_package/lexer.d b/test/dub_package/lexer.d new file mode 100755 index 000000000000..68f70d165f8b --- /dev/null +++ b/test/dub_package/lexer.d @@ -0,0 +1,31 @@ +#!/usr/bin/env dub +/+dub.sdl: +dependency "dmd" path="../.." ++/ +void main() +{ + import dmd.lexer; + import dmd.tokens; + + immutable expected = [ + TOKvoid, + TOKidentifier, + TOKlparen, + TOKrparen, + TOKlcurly, + TOKrcurly + ]; + + immutable sourceCode = "void test() {} // foobar"; + scope lexer = new Lexer("test", sourceCode.ptr, 0, sourceCode.length, 0, 0); + lexer.nextToken; + + TOK[] result; + + do + { + result ~= lexer.token.value; + } while (lexer.nextToken != TOKeof); + + assert(result == expected); +} diff --git a/test/dub_package/parser.d b/test/dub_package/parser.d old mode 100644 new mode 100755 index 7d8c77ed7e30..b997724ddeb0 --- a/test/dub_package/parser.d +++ b/test/dub_package/parser.d @@ -1,44 +1,8 @@ +#!/usr/bin/env dub /+dub.sdl: dependency "dmd" path="../.." +/ -// The tests in this module are highlevel and mostly indented to make sure all -// necessary modules are included in the Dub package. - void main() -{ -} - -// lexer -unittest -{ - import dmd.lexer; - import dmd.tokens; - - immutable expected = [ - TOKvoid, - TOKidentifier, - TOKlparen, - TOKrparen, - TOKlcurly, - TOKrcurly - ]; - - immutable sourceCode = "void test() {} // foobar"; - scope lexer = new Lexer("test", sourceCode.ptr, 0, sourceCode.length, 0, 0); - lexer.nextToken; - - TOK[] result; - - do - { - result ~= lexer.token.value; - } while (lexer.nextToken != TOKeof); - - assert(result == expected); -} - -// parser -unittest { import dmd.astbase; import dmd.parse; From 84752b69bee6ec561cde805e2fbdd7be5a8ac06f Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Thu, 14 Dec 2017 01:31:11 +0100 Subject: [PATCH 3/7] Test all DUB examples on Travis --- travis.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travis.sh b/travis.sh index efbed06b6db1..19cc030b9816 100755 --- a/travis.sh +++ b/travis.sh @@ -86,7 +86,9 @@ test_dmd() { test_dub_package() { source ~/dlang/*/activate # activate host compiler pushd test/dub_package - dub --single --build=unittest parser.d + for file in *.d ; do + dub --single "$file" + done popd deactivate } From 2f0b835f3fb2e798859561ed89b70c7a751cb78c Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Thu, 14 Dec 2017 02:01:15 +0100 Subject: [PATCH 4/7] Ignore all executables within test/dub_packages --- test/dub_package/.gitignore | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/dub_package/.gitignore b/test/dub_package/.gitignore index 5d51c5507b5d..9459d42ce679 100644 --- a/test/dub_package/.gitignore +++ b/test/dub_package/.gitignore @@ -1,6 +1,13 @@ +# Ignore all binary files (files without an extension) +# Start: Ignore everything +* +# Include directories +!*/ +# Include all files with extensions +!*.* + .dub docs.json __dummy.html *.o *.obj -/parser From e88073d788d51eb13183382e5d5719d379974211 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Thu, 14 Dec 2017 00:48:28 +0100 Subject: [PATCH 5/7] Add frontend module and example --- dub.sdl | 1 + src/dmd/frontend.d | 56 +++++++++++++ test/dub_package/frontend.d | 162 ++++++++++++++++++++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 src/dmd/frontend.d create mode 100755 test/dub_package/frontend.d diff --git a/dub.sdl b/dub.sdl index 2eac444644b5..f70cc98baefa 100644 --- a/dub.sdl +++ b/dub.sdl @@ -100,6 +100,7 @@ subPackage { "src/dmd/escape.d" \ "src/dmd/expression.d" \ "src/dmd/expressionsem.d" \ + "src/dmd/frontend.d" \ "src/dmd/func.d" \ "src/dmd/gluelayer.d" \ "src/dmd/hdrgen.d" \ diff --git a/src/dmd/frontend.d b/src/dmd/frontend.d new file mode 100644 index 000000000000..581f8dd9e5fb --- /dev/null +++ b/src/dmd/frontend.d @@ -0,0 +1,56 @@ +/** + * Compiler implementation of the + * $(LINK2 http://www.dlang.org, D programming language). + * + * This module contains high-level interfaces for interacting + with DMD as a library. + * + * Copyright: Copyright (c) 1999-2017 by The D Language Foundation, All Rights Reserved + * Authors: $(LINK2 http://www.digitalmars.com, Walter Bright) + * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) + * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/id.d, _id.d) + */ +module dmd.frontend; + +// Online documentation: https://dlang.org/phobos/dmd_frontend.html + +/** +Initializes the DMD compiler +*/ +void initDMD() +{ + import dmd.dmodule : Module; + import dmd.globals : global; + import dmd.id : Id; + import dmd.mtype : Type; + import dmd.target : Target; + import dmd.expression : Expression; + import dmd.objc : Objc; + import dmd.builtin : builtin_init; + + global._init; + + version(linux) + global.params.isLinux = 1; + else version(OSX) + global.params.isOSX = 1; + else version(FreeBSD) + global.params.isFreeBSD = 1; + else version(Windows) + global.params.isWindows = 1; + else version(Solaris) + global.params.isSolaris = 1; + else version(OpenBSD) + global.params.isOpenBSD = 1; + else + static assert(0, "OS not supported yet."); + + Type._init(); + Id.initialize(); + Module._init(); + Target._init(); + Expression._init(); + Objc._init(); + builtin_init(); +} + diff --git a/test/dub_package/frontend.d b/test/dub_package/frontend.d new file mode 100755 index 000000000000..2f8e26e3187d --- /dev/null +++ b/test/dub_package/frontend.d @@ -0,0 +1,162 @@ +#!/usr/bin/env dub +/+dub.sdl: +dependency "dmd" path="../.." ++/ +import std.stdio; + +// add import paths +void addImports(T)(T path) +{ + import dmd.globals : global; + import dmd.arraytypes : Strings; + + stderr.writefln("addImport: %s", path); + + Strings* res = new Strings(); + foreach (p; path) + { + import std.string : toStringz; + Strings* a = new Strings(); + a.push(p.toStringz); + res.append(a); + } + global.path = res; +} + +// finds a dmd.conf and parses it for import paths +auto findImportPaths() +{ + import std.file : exists, getcwd; + import std.string : fromStringz, toStringz; + import std.path : buildPath, buildNormalizedPath, dirName; + import std.process : env = environment, execute; + import dmd.dinifile : findConfFile; + import dmd.errors : fatal; + import std.algorithm, std.range, std.regex; + + auto dmdEnv = env.get("DMD", "dmd"); + auto whichDMD = execute(["which", dmdEnv]); + if (whichDMD.status != 0) + { + stderr.writeln("Can't find DMD."); + fatal; + } + + immutable dmdFilePath = whichDMD.output; + string iniFile; + + if (dmdEnv.canFind("ldmd")) + { + immutable ldcConfig = "ldc2.conf"; + immutable binDir = dmdFilePath.dirName; + // https://wiki.dlang.org/Using_LDC + auto ldcConfigs = [ + getcwd.buildPath(ldcConfig), + binDir.buildPath(ldcConfig), + binDir.dirName.buildPath("etc", ldcConfig), + "~/.ldc".buildPath(ldcConfig), + binDir.buildPath("etc", ldcConfig), + binDir.buildPath("etc", "ldc", ldcConfig), + "/etc".buildPath(ldcConfig), + "/etc/ldc".buildPath(ldcConfig), + ].filter!exists; + assert(!ldcConfigs.empty, "No ldc2.conf found"); + iniFile = ldcConfigs.front; + } + else + { + auto f = findConfFile(dmdFilePath.toStringz, "dmd.conf"); + iniFile = f.fromStringz.idup; + assert(iniFile.exists, "No dmd.conf found."); + } + + return File(iniFile, "r") + .byLineCopy + .map!(l => l.matchAll(`-I[^ "]+`.regex) + .joiner + .map!(a => a.drop(2) + .replace("%@P%", dmdFilePath.dirName) + .replace("%%ldcbinarypath%%", dmdFilePath.dirName))) + .joiner + .array + .sort + .uniq + .map!buildNormalizedPath; +} + +// test frontend +void main() +{ + import dmd.astcodegen : ASTCodegen; + import dmd.dmodule : Module; + import dmd.globals : global, Loc; + import dmd.frontend : initDMD; + import dmd.parse : Parser; + import dmd.statement : Identifier; + import dmd.tokens : TOKeof; + import dmd.id : Id; + + initDMD; + findImportPaths.addImports; + + auto parse(Module m, string code) + { + scope p = new Parser!ASTCodegen(m, code, false); + p.nextToken; // skip the initial token + auto members = p.parseModule; + assert(!p.errors, "Parsing error occurred."); + return members; + } + + Identifier id = Identifier.idPool("test"); + auto m = new Module("test.d", id, 0, 0); + m.members = parse(m, q{ + void foo() + { + foreach (i; 0..10) {} + } + }); + + void semantic() + { + import dmd.dsymbolsem : dsymbolSemantic; + import dmd.semantic : semantic2, semantic3; + + m.importedFrom = m; + m.importAll(null); + m.dsymbolSemantic(null); + m.semantic2(null); + m.semantic3(null); + } + semantic(); + + auto prettyPrint() + { + import dmd.root.outbuffer: OutBuffer; + import dmd.hdrgen : HdrGenState, PrettyPrintVisitor; + + OutBuffer buf = { doindent: 1 }; + HdrGenState hgs = { fullDump: 1 }; + scope PrettyPrintVisitor ppv = new PrettyPrintVisitor(&buf, &hgs); + m.accept(ppv); + return buf; + } + auto buf = prettyPrint(); + + auto expected =q{import object; +void foo() +{ + { + int __key3 = 0; + int __limit4 = 10; + for (; __key3 < __limit4; __key3 += 1) + { + int i = __key3; + } + } +} +}; + import std.string : replace, fromStringz; + auto generated = buf.extractData.fromStringz.replace("\t", " "); + assert(expected == generated, generated); +} From bb0161d1730d0408ce69e4c3f2a5856de7d7126a Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Sat, 16 Dec 2017 05:42:20 +0100 Subject: [PATCH 6/7] Use sourcePaths and excludeSourceFiles --- dub.sdl | 101 +++++--------------------------------------------------- 1 file changed, 8 insertions(+), 93 deletions(-) diff --git a/dub.sdl b/dub.sdl index f70cc98baefa..9c8a7c744dd8 100644 --- a/dub.sdl +++ b/dub.sdl @@ -53,101 +53,16 @@ subPackage { subPackage { name "frontend" targetType "library" + preGenerateCommands `cd "$${DUB_PACKAGE_DIR}" && ./config.sh generated/dub VERSION /etc` + stringImportPaths "generated/dub" stringImportPaths "res" versions "NoBackend" versions "GC" versions "NoMain" - sourcePaths - - sourceFiles \ - "src/dmd/access.d" \ - "src/dmd/aggregate.d" \ - "src/dmd/aliasthis.d" \ - "src/dmd/apply.d" \ - "src/dmd/argtypes.d" \ - "src/dmd/arrayop.d" \ - "src/dmd/arraytypes.d" \ - "src/dmd/astcodegen.d" \ - "src/dmd/attrib.d" \ - "src/dmd/blockexit.d" \ - "src/dmd/builtin.d" \ - "src/dmd/canthrow.d" \ - "src/dmd/clone.d" \ - "src/dmd/complex.d" \ - "src/dmd/cond.d" \ - "src/dmd/constfold.d" \ - "src/dmd/cppmangle.d" \ - "src/dmd/cppmanglewin.d" \ - "src/dmd/ctfeexpr.d" \ - "src/dmd/dcast.d" \ - "src/dmd/dclass.d" \ - "src/dmd/declaration.d" \ - "src/dmd/delegatize.d" \ - "src/dmd/denum.d" \ - "src/dmd/dimport.d" \ - "src/dmd/dinifile.d" \ - "src/dmd/dinterpret.d" \ - "src/dmd/dmacro.d" \ - "src/dmd/dmangle.d" \ - "src/dmd/dmodule.d" \ - "src/dmd/doc.d" \ - "src/dmd/dscope.d" \ - "src/dmd/dstruct.d" \ - "src/dmd/dsymbol.d" \ - "src/dmd/dsymbolsem.d" \ - "src/dmd/dtemplate.d" \ - "src/dmd/dversion.d" \ - "src/dmd/escape.d" \ - "src/dmd/expression.d" \ - "src/dmd/expressionsem.d" \ - "src/dmd/frontend.d" \ - "src/dmd/func.d" \ - "src/dmd/gluelayer.d" \ - "src/dmd/hdrgen.d" \ - "src/dmd/id.d" \ - "src/dmd/impcnvtab.d" \ - "src/dmd/imphint.d" \ - "src/dmd/init.d" \ - "src/dmd/initsem.d" \ - "src/dmd/inline.d" \ - "src/dmd/inlinecost.d" \ - "src/dmd/intrange.d" \ - "src/dmd/json.d" \ - "src/dmd/lib.d" \ - "src/dmd/link.d" \ - "src/dmd/mars.d" \ - "src/dmd/mtype.d" \ - "src/dmd/nogc.d" \ - "src/dmd/nspace.d" \ - "src/dmd/objc.d" \ - "src/dmd/opover.d" \ - "src/dmd/optimize.d" \ - "src/dmd/parse.d" \ - "src/dmd/parsetimevisitor.d" \ - "src/dmd/printast.d" \ - "src/dmd/safe.d" \ - "src/dmd/sapply.d" \ - "src/dmd/semantic.d" \ - "src/dmd/sideeffect.d" \ - "src/dmd/statement.d" \ - "src/dmd/statement_rewrite_walker.d" \ - "src/dmd/statementsem.d" \ - "src/dmd/staticassert.d" \ - "src/dmd/staticcond.d" \ - "src/dmd/target.d" \ - "src/dmd/templateparamsem.d" \ - "src/dmd/traits.d" \ - "src/dmd/typesem.d" \ - "src/dmd/typinf.d" \ - "src/dmd/utils.d" \ - "src/dmd/visitor.d" - - sourceFiles "src/dmd/scanelf.d" \ - "src/dmd/libelf.d" platform="linux" - - sourceFiles "src/dmd/scanmach.d" \ - "src/dmd/libmach.d" platform="osx" - - dependency "dmd:parser" version="*" - dependency "dmd:lexer" version="*" + sourcePaths "src/dmd" + excludedSourceFiles "src/dmd/backend/*" + excludedSourceFiles "src/dmd/{e2ir,eh,glue,iasm,objc_glue,s2ir,tocsym,toctype,toobj,todt,toir}.d" + excludedSourceFiles "src/dmd/{scan,lib}{mach,mscoff,omf}.d" platform="linux" + excludedSourceFiles "src/dmd/{scan,lib}{elf,mscoff,omf}.d" platform="osx" + excludedSourceFiles "src/dmd/{scan,lib}{elf,mach,mscoff}.d" platform="windows" } From 711f3653193337911baa633108032909904aa5f9 Mon Sep 17 00:00:00 2001 From: Sebastian Wilzbach Date: Tue, 19 Dec 2017 07:28:54 +0100 Subject: [PATCH 7/7] Don't run DUB frontend examples with GDC --- travis.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/travis.sh b/travis.sh index 19cc030b9816..94c838179c1d 100755 --- a/travis.sh +++ b/travis.sh @@ -85,11 +85,16 @@ test_dmd() { # test dub package test_dub_package() { source ~/dlang/*/activate # activate host compiler - pushd test/dub_package - for file in *.d ; do - dub --single "$file" - done - popd + # GDC's standard library is too old for some example scripts + if [ "${DMD:-dmd}" == "gdmd" ] ; then + echo "Skipping DUB examples on GDC." + else + pushd test/dub_package + for file in *.d ; do + dub --single "$file" + done + popd + fi deactivate }