From 95956db8a4fe743a1b03298b266352f050d6bb34 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Tue, 3 Apr 2018 19:36:34 -0700 Subject: [PATCH 01/22] Minimize compilation set --- index.js | 92 ++++++++++++------ package.json | 1 - profiler.js | 258 ++++++++++++++++++++++++++------------------------- 3 files changed, 194 insertions(+), 157 deletions(-) diff --git a/index.js b/index.js index a1edc29..0f8b8da 100644 --- a/index.js +++ b/index.js @@ -46,10 +46,10 @@ var compile = function(sources, options, callback) { process.removeListener("uncaughtException", solc_listener); } - // Ensure sources have operating system independent paths // i.e., convert backslashes to forward slashes; things like C: are left intact. var operatingSystemIndependentSources = {}; + var operatingSystemIndependentTargets = {}; var originalPathMappings = {}; Object.keys(sources).forEach(function(source) { @@ -65,30 +65,45 @@ var compile = function(sources, options, callback) { // Save the result operatingSystemIndependentSources[replacement] = sources[source]; + // Just substitute replacement for original in target case. It's + // a disposable subset of `sources` + if(options.compilationTargets && options.compilationTargets[source]){ + operatingSystemIndependentTargets[replacement] = sources[source]; + } + // Map the replacement back to the original source path. originalPathMappings[replacement] = source; }); + var defaultSelectors = { + "": [ + "legacyAST", + "ast" + ], + "*": [ + "abi", + "evm.bytecode.object", + "evm.bytecode.sourceMap", + "evm.deployedBytecode.object", + "evm.deployedBytecode.sourceMap" + ] + } + + // Specify compilation targets + var outputSelection = {}; + var targets = operatingSystemIndependentTargets; + var targetPaths = Object.keys(targets); + + (targetPaths.length) + ? targetPaths.forEach(key => outputSelection[key] = defaultSelectors) + : outputSelection["*"] = defaultSelectors; + var solcStandardInput = { language: "Solidity", sources: {}, settings: { optimizer: options.solc.optimizer, - outputSelection: { - "*": { - "": [ - "legacyAST", - "ast" - ], - "*": [ - "abi", - "evm.bytecode.object", - "evm.bytecode.sourceMap", - "evm.deployedBytecode.object", - "evm.deployedBytecode.sourceMap" - ] - }, - } + outputSelection: outputSelection, } }; @@ -151,6 +166,10 @@ var compile = function(sources, options, callback) { Object.keys(files_contracts).forEach(function(contract_name) { var contract = files_contracts[contract_name]; + // All source will have a key, but only the compiled source will have + // the evm output. + if (!Object.keys(contract.evm).length) return; + var contract_definition = { contract_name: contract_name, sourcePath: originalPathMappings[source_path], // Save original source path, not modified ones @@ -289,8 +308,10 @@ function orderABI(contract){ // quiet: Boolean. Suppress output. Defaults to false. // strict: Boolean. Return compiler warnings as errors. Defaults to false. compile.all = function(options, callback) { - var self = this; + find_contracts(options.contracts_directory, function(err, files) { + if (err) return callback(err) + options.paths = files; compile.with_dependencies(options, callback); }); @@ -319,6 +340,8 @@ compile.necessary = function(options, callback) { }; compile.with_dependencies = function(options, callback) { + var self = this; + options.logger = options.logger || console; options.contracts_directory = options.contracts_directory || process.cwd(); @@ -331,26 +354,37 @@ compile.with_dependencies = function(options, callback) { var config = Config.default().merge(options); - var self = this; Profiler.required_sources(config.with({ paths: options.paths, base_path: options.contracts_directory, resolver: options.resolver - }), function(err, result) { + }), (err, allSources, required) => { if (err) return callback(err); - if (options.quiet != true) { - Object.keys(result).sort().forEach(function(import_path) { - var display_path = import_path; - if (path.isAbsolute(import_path)) { - display_path = "." + path.sep + path.relative(options.working_directory, import_path); - } - options.logger.log("Compiling " + display_path + "..."); - }); - } + var hasTargets = Object.keys(required).length; + + (hasTargets) + ? self.display(required, options) + : self.display(allSources, options); - compile(result, options, callback); + options.compilationTargets = required; + compile(allSources, options, callback); }); }; +compile.display = function(paths, options){ + if (options.quiet != true) { + Object + .keys(paths) + .sort() + .forEach(contract => { + + if (path.isAbsolute(contract)) { + contract = "." + path.sep + path.relative(options.working_directory, contract); + } + options.logger.log("Compiling " + contract + "..."); + }); + } +} + module.exports = compile; diff --git a/package.json b/package.json index 01a9ab0..bf9845b 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,6 @@ "async": "^2.1.4", "colors": "^1.1.2", "debug": "^3.1.0", - "graphlib": "^2.1.1", "solc": "^0.4.21", "truffle-config": "^1.0.4", "truffle-contract-sources": "^0.0.1", diff --git a/profiler.js b/profiler.js index 9ecc9c2..b038f0a 100644 --- a/profiler.js +++ b/profiler.js @@ -4,8 +4,6 @@ var path = require("path"); var async = require("async"); var fs = require("fs"); -var Graph = require("graphlib").Graph; -var isAcyclic = require("graphlib/lib/alg").isAcyclic; var Parser = require("./parser"); var CompileError = require("./compileerror"); var expect = require("truffle-expect"); @@ -159,6 +157,8 @@ module.exports = { }); }, + // Returns the minimal set of sources to pass to solc as compilations targets, + // as well as the complete set of sources so solc can resolve the comp targets' imports. required_sources: function(options, callback) { var self = this; @@ -168,80 +168,160 @@ module.exports = { "resolver" ]); - var paths = this.convert_to_absolute_paths(options.paths, options.base_path); + var resolver = options.resolver; - function findRequiredSources(dependsGraph, done) { - var required = {}; + // Fetch the whole contract set + find_contracts(options.contracts_directory, (err, allPaths) => { + if(err) return callback(err); - function hasBeenTraversed(import_path) { - return required[import_path] != null; - } + // Solidity test files might have been injected. Include them in the known set. + options.paths.forEach(_path => { + if (!allPaths.includes(_path)) { + allPaths.push(_path) + } + }); - function include(import_path) { - //console.log("Including: " + file) + var updates = self.convert_to_absolute_paths(options.paths, options.base_path).sort(); + var allPaths = self.convert_to_absolute_paths(allPaths, options.base_path).sort(); - required[import_path] = dependsGraph.node(import_path); - } + var allSources = {}; + var compilationTargets = {}; - function walk_down(import_path) { - if (hasBeenTraversed(import_path)) { - return; - } + // Get all the source code + self.resolveAllSources(resolver, allPaths, (err, resolved) => { + if(err) return callback(err); - include(import_path); + // Generate hash of all sources including external packages - passed to solc inputs. + var resolvedPaths = Object.keys(resolved); + resolvedPaths.forEach(file => allSources[file] = resolved[file].body) - var dependencies = dependsGraph.successors(import_path); + // Exit w/out minimizing if we've been asked to compile everything, or nothing. + if (self.listsEqual(options.paths, allPaths)){ + return callback(null, allSources, {}); + } else if (!options.paths.length){ + return callback(null, {}, {}); + } - // console.log("At: " + import_path); - // console.log(" Dependencies: ", dependencies); + // Seed compilationTargets with known updates + updates.forEach(update => compilationTargets[update] = resolved[update].body); - if (dependencies.length > 0) { - dependencies.forEach(walk_down); - } - } + // While updates: dequeue + async.whilst(() => updates.length > 0, updateFinished => { + var currentUpdate = updates.shift(); + var files = allPaths.slice(); - function walk_from(import_path) { - if (hasBeenTraversed(import_path)) { - return; - } + // While files: dequeue and inspect their imports + async.whilst(() => files.length > 0, fileFinished => { - var ancestors = dependsGraph.predecessors(import_path); - var dependencies = dependsGraph.successors(import_path); + var currentFile = files.shift(); - // console.log("At: " + import_path); - // console.log(" Ancestors: ", ancestors); - // console.log(" Dependencies: ", dependencies); + // Ignore targets already selected. + if (compilationTargets[currentFile]){ + return fileFinished(); + } - include(import_path); + var imports; + try { + imports = self.getImports(currentFile, resolved[currentFile]); + } catch (err) { + err.message = "Error parsing " + currentFile + ": " + e.message; + return fileFinished(err); + } - if (ancestors && ancestors.length > 0) { - ancestors.forEach(walk_from); - } + // If file imports a compilation target, add it + // to list of updates and compilation targets + if (imports.includes(currentUpdate)){ + updates.push(currentFile); + compilationTargets[currentFile] = resolved[currentFile].body; + } - if (dependencies && dependencies.length > 0) { - dependencies.forEach(walk_down); - } - } + fileFinished(); + + }, err => updateFinished(err)); + }, err => (err) ? callback(err) : callback(null, allSources, compilationTargets)) + }) + }) + }, - paths.forEach(walk_from); + // Resolves sources in several async passes. For each resolved set it detects unknown + // imports from external packages and adds them to the set of files to resolve. + resolveAllSources: function(resolver, initialPaths, callback){ + var self = this; + var mapping = {}; + var allPaths = initialPaths.slice(); + + function generateMapping(finished){ + var promises = []; + + // Dequeue all the known paths, generating resolver promises, + // We'll add paths if we discover external package imports. + while(allPaths.length){ + var file = allPaths.shift(); + + var promise = new Promise((accept, reject)=> { + resolver.resolve(file, null, (err, body, absolutePath, source) => { + (err) + ? reject(err) + : accept({ file: absolutePath, body: body, source: source }); + }); + }); + promises.push(promise); + }; + + // Resolve everything known and add it to the map, then inspect each file's + // imports and add those to the list of paths to resolve if we don't have it. + Promise.all(promises).then(results => { + + // Generate the sources mapping + results.forEach(item => mapping[item.file] = Object.assign({}, item)); + + // Queue unknown imports for the next resolver cycle + while(results.length){ + var result = results.shift(); + + // Inspect the imports + var imports; + try { + imports = self.getImports(result.file, result); + } catch (err) { + err.message = "Error parsing " + result[file] + ": " + err.message; + return finished(err); + } - done(null, required); + // Detect unknown external packages / add them to the list of files to resolve + imports.forEach(item => (!mapping[item]) ? allPaths.push(item) : null) + }; + finished() + }).catch(err => { throw new Error(err) }); } - find_contracts(options.base_path, function(err, allPaths) { - if (err) return callback(err); + async.whilst( + () => allPaths.length, + generateMapping, + (err) => (err) ? callback(err) : callback(null, mapping) + ); + }, - // Include paths for Solidity .sols, specified in options. - allPaths = allPaths.concat(paths); + getImports: function(file, resolved){ + var self = this; - self.dependency_graph(allPaths, options.resolver, function(err, dependsGraph) { - if (err) return callback(err); + var imports = Parser.parseImports(resolved.body); - findRequiredSources(dependsGraph, callback); - }); + // Convert explicitly relative dependencies of modules back into module paths. + return imports.map(dependencyPath => { + return (self.isExplicitlyRelative(dependencyPath)) + ? resolved.source.resolve_dependency_path(file, dependencyPath) + : dependencyPath; }); }, + listsEqual: function(listA, listB){ + var a = listA.sort(); + var b = listB.sort(); + + return JSON.stringify(a) === JSON.stringify(b); + }, + convert_to_absolute_paths: function(paths, base) { var self = this; return paths.map(function(p) { @@ -260,82 +340,6 @@ module.exports = { return import_path.indexOf(".") == 0; }, - dependency_graph: function(paths, resolver, callback) { - var self = this; - - // Iterate through all the contracts looking for libraries and building a dependency graph - var dependsGraph = new Graph(); - - var imports_cache = {}; - - // For the purposes of determining correct error messages. - // The second array item denotes which path imported the current path. - // In the case of the paths passed in, there was none. - paths = paths.map(function(p) { - return [p, null]; - }); - - async.whilst(function() { - return paths.length > 0; - }, function(finished) { - var current = paths.shift(); - var import_path = current[0]; - var imported_from = current[1]; - - if (dependsGraph.hasNode(import_path) && imports_cache[import_path] != null) { - return finished(); - } - - resolver.resolve(import_path, imported_from, function(err, resolved_body, resolved_path, source) { - if (err) return finished(err); - - if (dependsGraph.hasNode(resolved_path) && imports_cache[resolved_path] != null) { - return finished(); - } - - // Add the contract to the depends graph. - dependsGraph.setNode(resolved_path, resolved_body); - - var imports; - - try { - imports = Parser.parseImports(resolved_body); - } catch (e) { - e.message = "Error parsing " + import_path + ": " + e.message; - return finished(e); - } - - // Convert explicitly relative dependencies of modules - // back into module paths. We also use this loop to update - // the graph edges. - imports = imports.map(function(dependency_path) { - // Convert explicitly relative paths - if (self.isExplicitlyRelative(dependency_path)) { - dependency_path = source.resolve_dependency_path(import_path, dependency_path); - } - - // Update graph edges - if (!dependsGraph.hasEdge(import_path, dependency_path)) { - dependsGraph.setEdge(import_path, dependency_path); - } - - // Return an array that denotes a new import and the path it was imported from. - return [dependency_path, import_path]; - }); - - imports_cache[import_path] = imports; - - Array.prototype.push.apply(paths, imports); - - finished(); - }); - }, - function(err) { - if (err) return callback(err); - callback(null, dependsGraph) - }); - }, - // Parse all source files in the directory and output the names of contracts and their source paths // directory can either be a directory or array of files. defined_contracts: function(directory, callback) { From fd114b89b1b8108160d5c61a7b401dd744234959 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 4 Apr 2018 17:59:32 -0700 Subject: [PATCH 02/22] Make compilationTargets an array --- index.js | 4 +++- profiler.js | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 0f8b8da..902b22d 100644 --- a/index.js +++ b/index.js @@ -31,6 +31,8 @@ var compile = function(sources, options, callback) { options.logger = console; } + var hasTargets = options.compilationTargets && options.compilationTargets.length; + expect.options(options, [ "contracts_directory", "solc" @@ -67,7 +69,7 @@ var compile = function(sources, options, callback) { // Just substitute replacement for original in target case. It's // a disposable subset of `sources` - if(options.compilationTargets && options.compilationTargets[source]){ + if(hasTargets && options.compilationTargets.includes(source)){ operatingSystemIndependentTargets[replacement] = sources[source]; } diff --git a/profiler.js b/profiler.js index b038f0a..7a3af12 100644 --- a/profiler.js +++ b/profiler.js @@ -185,7 +185,7 @@ module.exports = { var allPaths = self.convert_to_absolute_paths(allPaths, options.base_path).sort(); var allSources = {}; - var compilationTargets = {}; + var compilationTargets = []; // Get all the source code self.resolveAllSources(resolver, allPaths, (err, resolved) => { @@ -203,7 +203,7 @@ module.exports = { } // Seed compilationTargets with known updates - updates.forEach(update => compilationTargets[update] = resolved[update].body); + updates.forEach(update => compilationTargets.push(update)); // While updates: dequeue async.whilst(() => updates.length > 0, updateFinished => { @@ -216,7 +216,7 @@ module.exports = { var currentFile = files.shift(); // Ignore targets already selected. - if (compilationTargets[currentFile]){ + if (compilationTargets.includes(currentFile)){ return fileFinished(); } @@ -232,7 +232,7 @@ module.exports = { // to list of updates and compilation targets if (imports.includes(currentUpdate)){ updates.push(currentFile); - compilationTargets[currentFile] = resolved[currentFile].body; + compilationTargets.push(currentFile); } fileFinished(); From 8e27e6536fb7b7a2cf7d9088248f30f4ed62eb3e Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 4 Apr 2018 18:10:07 -0700 Subject: [PATCH 03/22] Improve comments --- index.js | 1 + profiler.js | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 902b22d..ca70fe2 100644 --- a/index.js +++ b/index.js @@ -92,6 +92,7 @@ var compile = function(sources, options, callback) { } // Specify compilation targets + // Each target uses defaultSelectors, defaulting to single target `*` if targets are unspecified var outputSelection = {}; var targets = operatingSystemIndependentTargets; var targetPaths = Object.keys(targets); diff --git a/profiler.js b/profiler.js index 7a3af12..4666f60 100644 --- a/profiler.js +++ b/profiler.js @@ -205,7 +205,10 @@ module.exports = { // Seed compilationTargets with known updates updates.forEach(update => compilationTargets.push(update)); - // While updates: dequeue + // While there are updated files in the queue, we take each one + // and search the entire file corpus to find any sources that import it. + // Those sources are added to list of compilation targets as well as + // the update queue because their own ancestors need to be discovered. async.whilst(() => updates.length > 0, updateFinished => { var currentUpdate = updates.shift(); var files = allPaths.slice(); From 2f5fc065290bb9c7d4c90a9f03aabf1508933e05 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 4 Apr 2018 20:01:39 -0700 Subject: [PATCH 04/22] Fix display for targets in array --- index.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index ca70fe2..64a8cb3 100644 --- a/index.js +++ b/index.js @@ -364,7 +364,7 @@ compile.with_dependencies = function(options, callback) { }), (err, allSources, required) => { if (err) return callback(err); - var hasTargets = Object.keys(required).length; + var hasTargets = required.length; (hasTargets) ? self.display(required, options) @@ -377,15 +377,15 @@ compile.with_dependencies = function(options, callback) { compile.display = function(paths, options){ if (options.quiet != true) { - Object - .keys(paths) - .sort() - .forEach(contract => { + if (!Array.isArray(paths)){ + paths = Object.keys(paths); + } - if (path.isAbsolute(contract)) { - contract = "." + path.sep + path.relative(options.working_directory, contract); - } - options.logger.log("Compiling " + contract + "..."); + paths.sort().forEach(contract => { + if (path.isAbsolute(contract)) { + contract = "." + path.sep + path.relative(options.working_directory, contract); + } + options.logger.log("Compiling " + contract + "..."); }); } } From a297be164a1fca8eed7ebb56d5cd123fc7c6d0e1 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Sat, 14 Apr 2018 19:54:31 -0700 Subject: [PATCH 05/22] Add CompilerProvider (solcjs) --- compilerProvider.js | 282 ++++++++++++++++++++++++++ index.js | 177 ++++++++-------- package.json | 5 + test/{ => sources}/ComplexOrdered.sol | 0 test/{ => sources}/InheritB.sol | 0 test/{ => sources}/MyContract.sol | 0 test/sources/NewPragma.sol | 9 + test/sources/OldPragmaFloat.sol | 9 + test/sources/OldPragmaPin.sol | 9 + test/{ => sources}/ShouldError.sol | 0 test/{ => sources}/SimpleOrdered.sol | 0 test/test_ordering.js | 6 +- test/test_parser.js | 10 +- test/test_provider.js | 209 +++++++++++++++++++ 14 files changed, 619 insertions(+), 97 deletions(-) create mode 100644 compilerProvider.js rename test/{ => sources}/ComplexOrdered.sol (100%) rename test/{ => sources}/InheritB.sol (100%) rename test/{ => sources}/MyContract.sol (100%) create mode 100644 test/sources/NewPragma.sol create mode 100644 test/sources/OldPragmaFloat.sol create mode 100644 test/sources/OldPragmaPin.sol rename test/{ => sources}/ShouldError.sol (100%) rename test/{ => sources}/SimpleOrdered.sol (100%) create mode 100644 test/test_provider.js diff --git a/compilerProvider.js b/compilerProvider.js new file mode 100644 index 0000000..14f600b --- /dev/null +++ b/compilerProvider.js @@ -0,0 +1,282 @@ +const path = require('path'); +const fs = require('fs'); +const request = require('request-promise'); +const requireFromString = require('require-from-string'); +const mkdirp = require('mkdirp'); + + +//------------------------------ Constructor/Config ------------------------------------------------ + +/** + * Constructor: + * @param {Object} _config (see `provider.config`) + */ +function CompilerProvider(_config){ + _config = _config || {}; + this.config = Object.assign(this.config, _config); +} + +/** + * Default configuration + * @type {Object} + */ +CompilerProvider.prototype.config = { + solc: null, + versionsUrl: 'https://solc-bin.ethereum.org/bin/list.json', + compilerUrl: 'https://solc-bin.ethereum.org/bin/', + compilerNpm: 'solc', + cache: true, + cachePath: '/var/lib/truffle/cache/solc/', +} + +//----------------------------------- Interface --------------------------------------------------- + +/** + * Loads solc from four possible locations: + * - local node_modules (param: ) + * - absolute path to a local solc (param: ) + * - a solc cache (param: && cache contains version) + * - a remote solc-bin source (param: && version not cached) + * + * @param {String} pathOrVersion [optional] version OR absolute path + * @return {Module} solc + */ +CompilerProvider.prototype.load = function(pathOrVersion){ + const self = this; + const solc = pathOrVersion || self.config.solc; + + return new Promise((accept, reject) => { + const useDefault = !solc; + const useLocal = !useDefault && self.isLocal(solc); + const useRemote = !useLocal; + + if (useDefault) return accept(self.getDefault()); + if (useLocal) return accept(self.getLocal(solc)); + if (useRemote) return accept(self.getByUrl(solc)); + }); +} + +/** + * Returns keys that can be used to specify which remote solc to fetch + * ``` + * latestRelease: "0.4.21", + * releases: ["0.4.21", ...], + * prereleases: ["0.4.22-nightly.2018.4.10+commit.27385d6d", ...] + * ``` + * @return {Object} See above + */ +CompilerProvider.prototype.getReleases = function(){ + return this + .getVersions() + .then(list => { + + // Prereleases + const prereleases = list + .builds + .filter(build => build['prerelease']) + .map(build => build['longVersion']); + + // Releases + const releases = Object.keys(list.releases); + + return { + prereleases: prereleases, + releases: releases, + latestRelease: list.latestRelease, + } + }); +} + +//------------------------------------ Getters ----------------------------------------------------- + +/** + * Gets solc from local `node_modules`. Equivalent to `require("solc")` + * @return {Module} solc + */ +CompilerProvider.prototype.getDefault = function(){ + const compiler = require(this.config.compilerNpm); + this.removeListener(); + return compiler; +} + +/** + * Gets an npm installed solc from specified absolute path. + * @param {String} localPath + * @return {Module} + */ +CompilerProvider.prototype.getLocal = function(localPath){ + const self = this; + let compiler; + + try { + compiler = require(localPath) + self.removeListener(); + } catch (err) { + throw self.errors('noPath', localPath); + } + + return compiler; +} + +/** + * Fetches solc versions object from remote solc-bin. This includes an array of build + * objects with detailed version info, an array of release version numbers + * and their terminal url component strings, and a latest version key with the + * same. + * @return {Object} versions + */ +CompilerProvider.prototype.getVersions = function(){ + const self = this; + url = self.config.versionsUrl; + + return request(url) + .then(list => JSON.parse(list)) + .catch(err => self.errors('noRequest', url, err)); +} + +/** + * Returns terminal url component for `version` from the versions object + * generated by `getVersions`. + * @param {String} version ex: "0.4.1", "0.4.16-nightly.2017.8.9+commit.81887bc7" + * @param {Object} allVersions (see `getVersions`) + * @return {String} url ex: "soljson-v0.4.21+commit.dfe3193c.js" + */ +CompilerProvider.prototype.getVersionUrl = function(version, allVersions){ + + if (allVersions.releases[version]) return allVersions.releases[version]; + + const isPrerelease = version.includes('nightly') || version.includes('commit'); + + if (isPrerelease) { + for (let build of allVersions.builds) { + const exists = build['prerelease'] === version || + build['build'] === version || + build['longVersion'] === version; + + if (exists) return build['path']; + } + } + + return null; +} + +/** + * Downloads solc specified by `version` after attempting retrieve it from cache on local machine, + * @param {String} version ex: "0.4.1", "0.4.16-nightly.2017.8.9+commit.81887bc7" + * @return {Module} solc + */ +CompilerProvider.prototype.getByUrl = function(version){ + const self = this; + + return self + .getVersions(self.config.versionsUrl) + .then(list => { + const location = self.getVersionUrl(version, list); + + if (!location) throw self.errors('noVersion', version); + + if (self.isCached(location)) return self.getFromCache(location); + + const url = self.config.compilerUrl + location; + + return request + .get(url) + .then(response => { + self.addToCache(response, location); + return self.compilerFromString(response); + }) + .catch(err => self.errors('noRequest', url, err)); + }); +} + +//------------------------------------ Utils ------------------------------------------------------- + +/** + * Returns true if file exists or `localPath` is an absolute path. + * @param {String} localPath + * @return {Boolean} + */ +CompilerProvider.prototype.isLocal = function(localPath){ + return fs.existsSync(localPath) || path.isAbsolute(localPath); +} + +/** + * Returns true if `fileName` exists in the cache. + * @param {String} fileName ex: "soljson-v0.4.21+commit.dfe3193c.js" + * @return {Boolean} + */ +CompilerProvider.prototype.isCached = function(fileName){ + return fs.existsSync(this.config.cachePath + fileName); +} + +/** + * Write to the cache at `config.cachePath`. Creates `cachePath` directory if + * does not exist. + * @param {String} code JS code string downloaded from solc-bin + * @param {String} fileName ex: "soljson-v0.4.21+commit.dfe3193c.js" + */ +CompilerProvider.prototype.addToCache = function(code, fileName){ + if (!this.config.cache) return; + + if (!fs.existsSync(this.config.cachePath)){ + mkdirp.sync(this.config.cachePath); + } + + fs.writeFileSync(this.config.cachePath + fileName, code); +} + +/** + * Retrieves usable solc module from cache + * @param {String} file ex: "soljson-v0.4.21+commit.dfe3193c.js" + * @return {Module} solc + */ +CompilerProvider.prototype.getFromCache = function(fileName){ + const cached = fs.readFileSync(this.config.cachePath + fileName, "utf-8"); + return this.compilerFromString(cached); +} + +/** + * Converts the JS code string obtained from solc-bin to usable node module. + * @param {String} code JS code + * @return {Module} solc + */ +CompilerProvider.prototype.compilerFromString = function(code){ + const solc = this.getDefault(); + const compiler = requireFromString(code); + return solc.setupMethods(compiler); +} + +/** + * Cleans up error listeners set (by solc?) when requiring it. (This code inherited from + * previous implementation, note to self - ask Tim about this) + */ +CompilerProvider.prototype.removeListener = function(){ + const listeners = process.listeners("uncaughtException"); + const execeptionHandler = listeners[listeners.length - 1]; + + if (execeptionHandler) { + process.removeListener("uncaughtException", execeptionHandler); + } +} + +/** + * Error formatter + * @param {String} kind descriptive key + * @param {String} input contextual info for error + * @param {Object} err [optional] additional error associated with this error + * @return {Error} + */ +CompilerProvider.prototype.errors = function(kind, input, err){ + const info = 'Run `truffle compile --list` to see available versions.' + + const kinds = { + noPath: "Could not find compiler at: " + input, + noVersion: "Could not find compiler version:\n" + input + ".\n" + info, + noString: "Expected string (`path` or `version`), got: " + input.toString(), + noRequest: "Failed to complete request to: " + input + ".\n\n" + err, + } + + return new Error(kinds[kind]); +} + +module.exports = CompilerProvider; diff --git a/index.js b/index.js index 64a8cb3..3ec5983 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,7 @@ var fs = require("fs"); var async = require("async"); var Profiler = require("./profiler"); var CompileError = require("./compileerror"); +var CompilerProvider = require("./compilerProvider"); var expect = require("truffle-expect"); var find_contracts = require("truffle-contract-sources"); var Config = require("truffle-config"); @@ -38,15 +39,7 @@ var compile = function(sources, options, callback) { "solc" ]); - // Load solc module only when compilation is actually required. - var solc = require("solc"); - // Clean up after solc. - var listeners = process.listeners("uncaughtException"); - var solc_listener = listeners[listeners.length - 1]; - if (solc_listener) { - process.removeListener("uncaughtException", solc_listener); - } // Ensure sources have operating system independent paths // i.e., convert backslashes to forward slashes; things like C: are left intact. @@ -121,110 +114,116 @@ var compile = function(sources, options, callback) { } }); - var result = solc.compileStandard(JSON.stringify(solcStandardInput)); + // Load solc module only when compilation is actually required. + var provider = new CompilerProvider(options.compiler); - var standardOutput = JSON.parse(result); + provider.load(options.solc).then(solc => { + var result = solc.compileStandard(JSON.stringify(solcStandardInput)); - var errors = standardOutput.errors || []; - var warnings = []; + var standardOutput = JSON.parse(result); - if (options.strict !== true) { - warnings = errors.filter(function(error) { - return error.severity == "warning"; - }); + var errors = standardOutput.errors || []; + var warnings = []; - errors = errors.filter(function(error) { - return error.severity != "warning"; - }); + if (options.strict !== true) { + warnings = errors.filter(function(error) { + return error.severity == "warning"; + }); + + errors = errors.filter(function(error) { + return error.severity != "warning"; + }); - if (options.quiet !== true && warnings.length > 0) { - options.logger.log(OS.EOL + "Compilation warnings encountered:" + OS.EOL); - options.logger.log(warnings.map(function(warning) { - return warning.formattedMessage; - }).join()); + if (options.quiet !== true && warnings.length > 0) { + options.logger.log(OS.EOL + "Compilation warnings encountered:" + OS.EOL); + options.logger.log(warnings.map(function(warning) { + return warning.formattedMessage; + }).join()); + } } - } - if (errors.length > 0) { - options.logger.log(""); - return callback(new CompileError(standardOutput.errors.map(function(error) { - return error.formattedMessage; - }).join())); - } + if (errors.length > 0) { + options.logger.log(""); + return callback(new CompileError(standardOutput.errors.map(function(error) { + return error.formattedMessage; + }).join())); + } - var contracts = standardOutput.contracts; + var contracts = standardOutput.contracts; - var files = []; - Object.keys(standardOutput.sources).forEach(function(filename) { - var source = standardOutput.sources[filename]; - files[source.id] = filename; - }); + var files = []; + Object.keys(standardOutput.sources).forEach(function(filename) { + var source = standardOutput.sources[filename]; + files[source.id] = filename; + }); - var returnVal = {}; - - // This block has comments in it as it's being prepared for solc > 0.4.10 - Object.keys(contracts).forEach(function(source_path) { - var files_contracts = contracts[source_path]; - - Object.keys(files_contracts).forEach(function(contract_name) { - var contract = files_contracts[contract_name]; - - // All source will have a key, but only the compiled source will have - // the evm output. - if (!Object.keys(contract.evm).length) return; - - var contract_definition = { - contract_name: contract_name, - sourcePath: originalPathMappings[source_path], // Save original source path, not modified ones - source: operatingSystemIndependentSources[source_path], - sourceMap: contract.evm.bytecode.sourceMap, - deployedSourceMap: contract.evm.deployedBytecode.sourceMap, - legacyAST: standardOutput.sources[source_path].legacyAST, - ast: standardOutput.sources[source_path].ast, - abi: contract.abi, - bytecode: "0x" + contract.evm.bytecode.object, - deployedBytecode: "0x" + contract.evm.deployedBytecode.object, - unlinked_binary: "0x" + contract.evm.bytecode.object, // deprecated - compiler: { - "name": "solc", - "version": solc.version() + var returnVal = {}; + + // This block has comments in it as it's being prepared for solc > 0.4.10 + Object.keys(contracts).forEach(function(source_path) { + var files_contracts = contracts[source_path]; + + Object.keys(files_contracts).forEach(function(contract_name) { + var contract = files_contracts[contract_name]; + + // All source will have a key, but only the compiled source will have + // the evm output. + if (!Object.keys(contract.evm).length) return; + + var contract_definition = { + contract_name: contract_name, + sourcePath: originalPathMappings[source_path], // Save original source path, not modified ones + source: operatingSystemIndependentSources[source_path], + sourceMap: contract.evm.bytecode.sourceMap, + deployedSourceMap: contract.evm.deployedBytecode.sourceMap, + legacyAST: standardOutput.sources[source_path].legacyAST, + ast: standardOutput.sources[source_path].ast, + abi: contract.abi, + bytecode: "0x" + contract.evm.bytecode.object, + deployedBytecode: "0x" + contract.evm.deployedBytecode.object, + unlinked_binary: "0x" + contract.evm.bytecode.object, // deprecated + compiler: { + "name": "solc", + "version": solc.version() + } } - } - // Reorder ABI so functions are listed in the order they appear - // in the source file. Solidity tests need to execute in their expected sequence. - contract_definition.abi = orderABI(contract_definition); + // Reorder ABI so functions are listed in the order they appear + // in the source file. Solidity tests need to execute in their expected sequence. + contract_definition.abi = orderABI(contract_definition); - // Go through the link references and replace them with older-style - // identifiers. We'll do this until we're ready to making a breaking - // change to this code. - Object.keys(contract.evm.bytecode.linkReferences).forEach(function(file_name) { - var fileLinks = contract.evm.bytecode.linkReferences[file_name]; + // Go through the link references and replace them with older-style + // identifiers. We'll do this until we're ready to making a breaking + // change to this code. + Object.keys(contract.evm.bytecode.linkReferences).forEach(function(file_name) { + var fileLinks = contract.evm.bytecode.linkReferences[file_name]; - Object.keys(fileLinks).forEach(function(library_name) { - var linkReferences = fileLinks[library_name] || []; + Object.keys(fileLinks).forEach(function(library_name) { + var linkReferences = fileLinks[library_name] || []; - contract_definition.bytecode = replaceLinkReferences(contract_definition.bytecode, linkReferences, library_name); - contract_definition.unlinked_binary = replaceLinkReferences(contract_definition.unlinked_binary, linkReferences, library_name); + contract_definition.bytecode = replaceLinkReferences(contract_definition.bytecode, linkReferences, library_name); + contract_definition.unlinked_binary = replaceLinkReferences(contract_definition.unlinked_binary, linkReferences, library_name); + }); }); - }); - // Now for the deployed bytecode - Object.keys(contract.evm.deployedBytecode.linkReferences).forEach(function(file_name) { - var fileLinks = contract.evm.deployedBytecode.linkReferences[file_name]; + // Now for the deployed bytecode + Object.keys(contract.evm.deployedBytecode.linkReferences).forEach(function(file_name) { + var fileLinks = contract.evm.deployedBytecode.linkReferences[file_name]; - Object.keys(fileLinks).forEach(function(library_name) { - var linkReferences = fileLinks[library_name] || []; + Object.keys(fileLinks).forEach(function(library_name) { + var linkReferences = fileLinks[library_name] || []; - contract_definition.deployedBytecode = replaceLinkReferences(contract_definition.deployedBytecode, linkReferences, library_name); + contract_definition.deployedBytecode = replaceLinkReferences(contract_definition.deployedBytecode, linkReferences, library_name); + }); }); - }); - returnVal[contract_name] = contract_definition; + returnVal[contract_name] = contract_definition; + }); }); - }); - callback(null, returnVal, files); + callback(null, returnVal, files); + }) + .catch(callback); }; function replaceLinkReferences(bytecode, linkReferences, libraryName) { diff --git a/package.json b/package.json index 747fe83..1d9fc44 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,10 @@ "async": "^2.1.4", "colors": "^1.1.2", "debug": "^3.1.0", + "mkdirp": "^0.5.1", + "request": "^2.85.0", + "request-promise": "^4.2.2", + "require-from-string": "^2.0.2", "solc": "0.4.21", "truffle-config": "^1.0.4", "truffle-contract-sources": "^0.0.1", @@ -15,6 +19,7 @@ }, "devDependencies": { "mocha": "^3.5.3", + "tmp": "0.0.33", "truffle-resolver": "2.0.0" }, "scripts": { diff --git a/test/ComplexOrdered.sol b/test/sources/ComplexOrdered.sol similarity index 100% rename from test/ComplexOrdered.sol rename to test/sources/ComplexOrdered.sol diff --git a/test/InheritB.sol b/test/sources/InheritB.sol similarity index 100% rename from test/InheritB.sol rename to test/sources/InheritB.sol diff --git a/test/MyContract.sol b/test/sources/MyContract.sol similarity index 100% rename from test/MyContract.sol rename to test/sources/MyContract.sol diff --git a/test/sources/NewPragma.sol b/test/sources/NewPragma.sol new file mode 100644 index 0000000..82916e5 --- /dev/null +++ b/test/sources/NewPragma.sol @@ -0,0 +1,9 @@ +pragma solidity ^0.4.21; + +contract NewPragma { + uint x; + + function NewPragma() public { + x = 5; + } +} \ No newline at end of file diff --git a/test/sources/OldPragmaFloat.sol b/test/sources/OldPragmaFloat.sol new file mode 100644 index 0000000..d505811 --- /dev/null +++ b/test/sources/OldPragmaFloat.sol @@ -0,0 +1,9 @@ +pragma solidity ^0.4.15; + +contract OldPragmaFloat { + uint x; + + function OldPragmaFloat() public { + x = 7; + } +} \ No newline at end of file diff --git a/test/sources/OldPragmaPin.sol b/test/sources/OldPragmaPin.sol new file mode 100644 index 0000000..e3c8fa3 --- /dev/null +++ b/test/sources/OldPragmaPin.sol @@ -0,0 +1,9 @@ +pragma solidity 0.4.15; + +contract OldPragmaPin { + uint x; + + function OldPragmaPin() public { + x = 7; + } +} \ No newline at end of file diff --git a/test/ShouldError.sol b/test/sources/ShouldError.sol similarity index 100% rename from test/ShouldError.sol rename to test/sources/ShouldError.sol diff --git a/test/SimpleOrdered.sol b/test/sources/SimpleOrdered.sol similarity index 100% rename from test/SimpleOrdered.sol rename to test/sources/SimpleOrdered.sol diff --git a/test/test_ordering.js b/test/test_ordering.js index e495f60..c1b3fd3 100644 --- a/test/test_ordering.js +++ b/test/test_ordering.js @@ -12,9 +12,9 @@ describe("Compile", function() { describe("ABI Ordering", function(){ before("get code", function() { - simpleOrderedSource = fs.readFileSync(path.join(__dirname, "SimpleOrdered.sol"), "utf-8"); - complexOrderedSource = fs.readFileSync(path.join(__dirname, "ComplexOrdered.sol"), "utf-8"); - inheritedSource = fs.readFileSync(path.join(__dirname, "InheritB.sol"), "utf-8"); + simpleOrderedSource = fs.readFileSync(path.join(__dirname, "./sources/SimpleOrdered.sol"), "utf-8"); + complexOrderedSource = fs.readFileSync(path.join(__dirname, "./sources/ComplexOrdered.sol"), "utf-8"); + inheritedSource = fs.readFileSync(path.join(__dirname, "./sources/InheritB.sol"), "utf-8"); }); // Ordered.sol's methods are ordered semantically. diff --git a/test/test_parser.js b/test/test_parser.js index b88ba8a..68419b2 100644 --- a/test/test_parser.js +++ b/test/test_parser.js @@ -8,8 +8,8 @@ describe("Parser", function() { var erroneousSource = null; before("get code", function() { - source = fs.readFileSync(path.join(__dirname, "MyContract.sol"), "utf-8"); - erroneousSource = fs.readFileSync(path.join(__dirname, "ShouldError.sol"), "utf-8"); + source = fs.readFileSync(path.join(__dirname, "./sources/MyContract.sol"), "utf-8"); + erroneousSource = fs.readFileSync(path.join(__dirname, "./sources/ShouldError.sol"), "utf-8"); }); it("should return correct imports", function() { @@ -49,10 +49,10 @@ describe("Parser", function() { var output = Parser.parse(source); assert.deepEqual(output.contracts, ["MyContract", "SomeInterface", "SomeLibrary"]); - assert(output.ast.nodes.length > 0); - + assert(output.ast.nodes.length > 0); + // The above assert means we at least got some kind of AST. - // Is there something we specifically need here? + // Is there something we specifically need here? }); it("should throw an error when parsing completely if there's an actual parse error", function() { diff --git a/test/test_provider.js b/test/test_provider.js new file mode 100644 index 0000000..c702a4e --- /dev/null +++ b/test/test_provider.js @@ -0,0 +1,209 @@ +const fs = require("fs"); +const tmp = require("tmp"); +const path = require("path"); +const solc = require("solc"); +const assert = require("assert"); +const compile = require("../index"); +const CompilerProvider = require('../compilerProvider'); + +function waitSecond() { + return new Promise((resolve, reject) => setTimeout(() => resolve(), 1250)); +} + +describe.only('CompilerProvider', function(){ + let provider; + + describe('getters', function(){ + + before(() => provider = new CompilerProvider()); + + it('getVersions: should return versions object', async function(){ + const list = await provider.getVersions(); + assert(list.releases !== undefined); + }); + + it('getVersionUrl: should return a url component', async function(){ + const list = await provider.getVersions(); + + let input = '0.4.21'; + let expected = 'soljson-v0.4.21+commit.dfe3193c.js'; + + location = await provider.getVersionUrl(input, list); + assert(location === expected, 'Should locate by version'); + + input = 'nightly.2018.4.12'; + expected = 'soljson-v0.4.22-nightly.2018.4.12+commit.c3dc67d0.js'; + + location = await provider.getVersionUrl(input, list); + assert(location === expected, 'Should locate by nightly'); + + input = 'commit.c3dc67d0'; + expected = 'soljson-v0.4.22-nightly.2018.4.12+commit.c3dc67d0.js'; + + location = await provider.getVersionUrl(input, list); + assert(location === expected, 'Should locate by commit'); + + input = '0.4.55.77-fantasy-solc'; + expected = null; + + location = await provider.getVersionUrl(input, list); + assert(location === null, 'Should return null if not found') + }); + + it('getReleases: should return a `releases` object', async function(){ + const list = await provider.getVersions(); + const releases = await provider.getReleases(); + const firstSolc = "0.1.3-nightly.2015.9.25+commit.4457170" + + assert(releases.prereleases[0] === firstSolc, 'Should return prereleases'); + assert(releases.releases[0] === releases.latestRelease, 'Should return releases/latestRelease'); + }); + }); + + describe('integration', function(){ + this.timeout(40000); + let newPragmaSource; // floating at ^0.4.21 + let oldPragmaSource; // pinned at 0.4.15 + + const options = { + contracts_directory: '', + solc: '', + quiet: true, + }; + + before("get code", function() { + const newPragma = fs.readFileSync(path.join(__dirname, "./sources/NewPragma.sol"), "utf-8"); + const oldPragmaPin = fs.readFileSync(path.join(__dirname, "./sources/OldPragmaPin.sol"), "utf-8"); + const oldPragmaFloat = fs.readFileSync(path.join(__dirname, "./sources/OldPragmaFloat.sol"), "utf-8"); + + newPragmaSource = { "NewPragma.sol": newPragma}; + oldPragmaPinSource = { "OldPragmaPin.sol": oldPragmaPin}; + oldPragmaFloatSource = { "OldPragmaPin.sol": oldPragmaFloat}; + }); + + it('compiles w/ default solc if no compiler specified (float)', function(done){ + options.compiler = { cache: false }; + + compile(newPragmaSource, options, (err, result) => { + assert(err == null); + assert(result['NewPragma'].contract_name === 'NewPragma'); + done(); + }); + }); + + it('compiles w/ remote solc when options specify release (pinned)', function(done){ + options.compiler = { + cache: false, + solc: "0.4.15" + }; + + compile(oldPragmaPinSource, options, (err, result) => { + assert(err == null); + assert(result['OldPragmaPin'].contract_name === 'OldPragmaPin'); + done(); + }); + }); + + it('compiles w/ remote solc when options specify prerelease (float)', function(done){ + // An 0.4.16 prerelease for 0.4.15 + options.compiler = { + cache: false, + solc: "0.4.16-nightly.2017.8.9+commit.81887bc7" + }; + + compile(oldPragmaFloatSource, options, (err, result) => { + assert(err == null); + assert(result['OldPragmaFloat'].contract_name === 'OldPragmaFloat'); + done(); + }); + }); + + it('errors when specified release does not exist', function(done){ + options.compiler = { + cache: false, + solc: "0.4.55.77-fantasy-solc" + }; + + compile(newPragmaSource, options, (err, result) => { + assert(err.message.includes('Could not find compiler version')); + done(); + }); + }); + + it('compiles w/ local path solc when options specify path', function(done){ + const pathToSolc = path.join(__dirname, "../node_modules/solc/index.js"); + + options.compiler = { + cache: false, + solc: pathToSolc + }; + + compile(newPragmaSource, options, (err, result) => { + assert(err == null); + assert(result['NewPragma'].contract_name === 'NewPragma'); + done(); + }); + }); + + it('errors when specified path does not exist', function(done){ + const pathToSolc = path.join(__dirname, "../solidity-warehouse/solc/index.js"); + options.compiler = { + cache: false, + solc: pathToSolc + }; + + compile(newPragmaSource, options, (err, result) => { + assert(err.message.includes('Could not find compiler at:')); + done(); + }); + }); + + it('caches releases and uses them if available', function(done){ + tmp.dir((err, dir) => { + if(err) return done(err); + + const truffleCacheDir = dir + "/truffle/solc/cache/"; + const expectedCache = truffleCacheDir + 'soljson-v0.4.21+commit.dfe3193c.js'; + + options.compiler = { + cache: true, + cachePath: truffleCacheDir, + solc: "0.4.21" + }; + + // Run compiler, expecting solc to be downloaded and cached. + compile(newPragmaSource, options, (err, result) => { + assert(err === null); + assert(fs.existsSync(expectedCache)); + + // Get cached solc access time + const initialAccessTime = fs.statSync(expectedCache).atime.getTime(); + + // Wait a second and recompile, verifying that the cached solc + // got accessed / ran ok. + waitSecond().then(() => { + + compile(newPragmaSource, options, (err, result) => { + const finalAccessTime = fs.statSync(expectedCache).atime.getTime(); + + assert(err == null); + assert(result['NewPragma'].contract_name === 'NewPragma'); + assert(initialAccessTime < finalAccessTime); + done(); + }); + + }).catch(done); + }); + }); + }); + }); +}); + + + + + + + + + From ccc34619b7dafdee90bdd3f755c2160fe7653319 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Sun, 15 Apr 2018 09:46:51 -0700 Subject: [PATCH 06/22] Use CompilerProvider to parse imports --- compilerProvider.js | 24 +++++----- index.js | 2 +- parser.js | 5 +- profiler.js | 104 ++++++++++++++++++++++-------------------- test/test_parser.js | 15 ++++-- test/test_provider.js | 12 ++--- 6 files changed, 86 insertions(+), 76 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index 14f600b..fe94318 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -38,12 +38,12 @@ CompilerProvider.prototype.config = { * - a solc cache (param: && cache contains version) * - a remote solc-bin source (param: && version not cached) * - * @param {String} pathOrVersion [optional] version OR absolute path + * @param {String} options [optional] options to pass to native solc binary * @return {Module} solc */ -CompilerProvider.prototype.load = function(pathOrVersion){ +CompilerProvider.prototype.load = function(options){ const self = this; - const solc = pathOrVersion || self.config.solc; + const solc = self.config.solc; return new Promise((accept, reject) => { const useDefault = !solc; @@ -121,7 +121,7 @@ CompilerProvider.prototype.getLocal = function(localPath){ /** * Fetches solc versions object from remote solc-bin. This includes an array of build * objects with detailed version info, an array of release version numbers - * and their terminal url component strings, and a latest version key with the + * and their terminal url segment strings, and a latest version key with the * same. * @return {Object} versions */ @@ -135,13 +135,13 @@ CompilerProvider.prototype.getVersions = function(){ } /** - * Returns terminal url component for `version` from the versions object + * Returns terminal url segment for `version` from the versions object * generated by `getVersions`. * @param {String} version ex: "0.4.1", "0.4.16-nightly.2017.8.9+commit.81887bc7" * @param {Object} allVersions (see `getVersions`) * @return {String} url ex: "soljson-v0.4.21+commit.dfe3193c.js" */ -CompilerProvider.prototype.getVersionUrl = function(version, allVersions){ +CompilerProvider.prototype.getVersionUrlSegment = function(version, allVersions){ if (allVersions.releases[version]) return allVersions.releases[version]; @@ -170,19 +170,19 @@ CompilerProvider.prototype.getByUrl = function(version){ return self .getVersions(self.config.versionsUrl) - .then(list => { - const location = self.getVersionUrl(version, list); + .then(allVersions => { + const file = self.getVersionUrlSegment(version, allVersions); - if (!location) throw self.errors('noVersion', version); + if (!file) throw self.errors('noVersion', version); - if (self.isCached(location)) return self.getFromCache(location); + if (self.isCached(file)) return self.getFromCache(file); - const url = self.config.compilerUrl + location; + const url = self.config.compilerUrl + file; return request .get(url) .then(response => { - self.addToCache(response, location); + self.addToCache(response, file); return self.compilerFromString(response); }) .catch(err => self.errors('noRequest', url, err)); diff --git a/index.js b/index.js index 3ec5983..b7b5659 100644 --- a/index.js +++ b/index.js @@ -117,7 +117,7 @@ var compile = function(sources, options, callback) { // Load solc module only when compilation is actually required. var provider = new CompilerProvider(options.compiler); - provider.load(options.solc).then(solc => { + provider.load().then(solc => { var result = solc.compileStandard(JSON.stringify(solcStandardInput)); var standardOutput = JSON.parse(result); diff --git a/parser.js b/parser.js index ec74935..383da84 100644 --- a/parser.js +++ b/parser.js @@ -1,5 +1,4 @@ var CompileError = require("./compileerror"); -var solc = require("solc"); var fs = require("fs"); var path = require("path"); @@ -33,7 +32,7 @@ module.exports = { remappings.push(pkg + "/=" + path.join(installedContractsDir, pkg, 'contracts', '/')); } } - + return remappings; } @@ -97,7 +96,7 @@ module.exports = { }, // This needs to be fast! It is fast (as of this writing). Keep it fast! - parseImports: function(body) { + parseImports: function(body, solc) { var self = this; // WARNING: Kind of a hack (an expedient one). diff --git a/profiler.js b/profiler.js index 4666f60..a93cbd0 100644 --- a/profiler.js +++ b/profiler.js @@ -6,6 +6,7 @@ var async = require("async"); var fs = require("fs"); var Parser = require("./parser"); var CompileError = require("./compileerror"); +var CompilerProvider = require("./compilerProvider"); var expect = require("truffle-expect"); var find_contracts = require("truffle-contract-sources"); var debug = require("debug")("compile:profiler"); @@ -187,68 +188,73 @@ module.exports = { var allSources = {}; var compilationTargets = []; - // Get all the source code - self.resolveAllSources(resolver, allPaths, (err, resolved) => { - if(err) return callback(err); + // Load compiler + var provider = new CompilerProvider(options.compiler) + provider.load().then(solc => { - // Generate hash of all sources including external packages - passed to solc inputs. - var resolvedPaths = Object.keys(resolved); - resolvedPaths.forEach(file => allSources[file] = resolved[file].body) + // Get all the source code + self.resolveAllSources(resolver, allPaths, solc, (err, resolved) => { + if(err) return callback(err); - // Exit w/out minimizing if we've been asked to compile everything, or nothing. - if (self.listsEqual(options.paths, allPaths)){ - return callback(null, allSources, {}); - } else if (!options.paths.length){ - return callback(null, {}, {}); - } + // Generate hash of all sources including external packages - passed to solc inputs. + var resolvedPaths = Object.keys(resolved); + resolvedPaths.forEach(file => allSources[file] = resolved[file].body) - // Seed compilationTargets with known updates - updates.forEach(update => compilationTargets.push(update)); + // Exit w/out minimizing if we've been asked to compile everything, or nothing. + if (self.listsEqual(options.paths, allPaths)){ + return callback(null, allSources, {}); + } else if (!options.paths.length){ + return callback(null, {}, {}); + } - // While there are updated files in the queue, we take each one - // and search the entire file corpus to find any sources that import it. - // Those sources are added to list of compilation targets as well as - // the update queue because their own ancestors need to be discovered. - async.whilst(() => updates.length > 0, updateFinished => { - var currentUpdate = updates.shift(); - var files = allPaths.slice(); + // Seed compilationTargets with known updates + updates.forEach(update => compilationTargets.push(update)); - // While files: dequeue and inspect their imports - async.whilst(() => files.length > 0, fileFinished => { + // While there are updated files in the queue, we take each one + // and search the entire file corpus to find any sources that import it. + // Those sources are added to list of compilation targets as well as + // the update queue because their own ancestors need to be discovered. + async.whilst(() => updates.length > 0, updateFinished => { + var currentUpdate = updates.shift(); + var files = allPaths.slice(); - var currentFile = files.shift(); + // While files: dequeue and inspect their imports + async.whilst(() => files.length > 0, fileFinished => { - // Ignore targets already selected. - if (compilationTargets.includes(currentFile)){ - return fileFinished(); - } + var currentFile = files.shift(); - var imports; - try { - imports = self.getImports(currentFile, resolved[currentFile]); - } catch (err) { - err.message = "Error parsing " + currentFile + ": " + e.message; - return fileFinished(err); - } + // Ignore targets already selected. + if (compilationTargets.includes(currentFile)){ + return fileFinished(); + } - // If file imports a compilation target, add it - // to list of updates and compilation targets - if (imports.includes(currentUpdate)){ - updates.push(currentFile); - compilationTargets.push(currentFile); - } + var imports; + try { + imports = self.getImports(currentFile, resolved[currentFile], solc); + } catch (err) { + err.message = "Error parsing " + currentFile + ": " + e.message; + return fileFinished(err); + } - fileFinished(); + // If file imports a compilation target, add it + // to list of updates and compilation targets + if (imports.includes(currentUpdate)){ + updates.push(currentFile); + compilationTargets.push(currentFile); + } - }, err => updateFinished(err)); - }, err => (err) ? callback(err) : callback(null, allSources, compilationTargets)) - }) + fileFinished(); + + }, err => updateFinished(err)); + }, err => (err) ? callback(err) : callback(null, allSources, compilationTargets)) + }) + }).catch(callback) }) }, // Resolves sources in several async passes. For each resolved set it detects unknown // imports from external packages and adds them to the set of files to resolve. - resolveAllSources: function(resolver, initialPaths, callback){ + resolveAllSources: function(resolver, initialPaths, solc, callback){ var self = this; var mapping = {}; var allPaths = initialPaths.slice(); @@ -285,7 +291,7 @@ module.exports = { // Inspect the imports var imports; try { - imports = self.getImports(result.file, result); + imports = self.getImports(result.file, result, solc); } catch (err) { err.message = "Error parsing " + result[file] + ": " + err.message; return finished(err); @@ -305,10 +311,10 @@ module.exports = { ); }, - getImports: function(file, resolved){ + getImports: function(file, resolved, solc){ var self = this; - var imports = Parser.parseImports(resolved.body); + var imports = Parser.parseImports(resolved.body, solc); // Convert explicitly relative dependencies of modules back into module paths. return imports.map(dependencyPath => { diff --git a/test/test_parser.js b/test/test_parser.js index 68419b2..8c9301e 100644 --- a/test/test_parser.js +++ b/test/test_parser.js @@ -1,19 +1,24 @@ var fs = require("fs"); var path = require("path"); var Parser = require("../parser"); +var CompilerProvider = require("../compilerProvider") var assert = require("assert"); describe("Parser", function() { var source = null; var erroneousSource = null; + var solc; - before("get code", function() { + before("get code", async function() { source = fs.readFileSync(path.join(__dirname, "./sources/MyContract.sol"), "utf-8"); erroneousSource = fs.readFileSync(path.join(__dirname, "./sources/ShouldError.sol"), "utf-8"); + + const provider = new CompilerProvider(); + solc = await provider.load(); }); it("should return correct imports", function() { - var imports = Parser.parseImports(source); + var imports = Parser.parseImports(source, solc); // Note that this test is important because certain parts of the solidity // output cuts off path prefixes like "./" and "../../../". If we get the @@ -31,7 +36,7 @@ describe("Parser", function() { it("should throw an error when parsing imports if there's an actual parse error", function() { var error = null; try { - Parser.parseImports(erroneousSource); + Parser.parseImports(erroneousSource, solc); } catch(e) { error = e; } @@ -43,7 +48,7 @@ describe("Parser", function() { assert(error.message.indexOf("Expected pragma, import directive or contract") >= 0); }); - it("should return a full AST when parsed, even when dependencies don't exist", function() { + it.skip("should return a full AST when parsed, even when dependencies don't exist", function() { this.timeout(4000); var output = Parser.parse(source); @@ -55,7 +60,7 @@ describe("Parser", function() { // Is there something we specifically need here? }); - it("should throw an error when parsing completely if there's an actual parse error", function() { + it.skip("should throw an error when parsing completely if there's an actual parse error", function() { var error = null; try { Parser.parse(erroneousSource); diff --git a/test/test_provider.js b/test/test_provider.js index c702a4e..f16c92f 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -10,7 +10,7 @@ function waitSecond() { return new Promise((resolve, reject) => setTimeout(() => resolve(), 1250)); } -describe.only('CompilerProvider', function(){ +describe('CompilerProvider', function(){ let provider; describe('getters', function(){ @@ -22,31 +22,31 @@ describe.only('CompilerProvider', function(){ assert(list.releases !== undefined); }); - it('getVersionUrl: should return a url component', async function(){ + it('getVersionUrlSegment: should return a JS file name', async function(){ const list = await provider.getVersions(); let input = '0.4.21'; let expected = 'soljson-v0.4.21+commit.dfe3193c.js'; - location = await provider.getVersionUrl(input, list); + location = await provider.getVersionUrlSegment(input, list); assert(location === expected, 'Should locate by version'); input = 'nightly.2018.4.12'; expected = 'soljson-v0.4.22-nightly.2018.4.12+commit.c3dc67d0.js'; - location = await provider.getVersionUrl(input, list); + location = await provider.getVersionUrlSegment(input, list); assert(location === expected, 'Should locate by nightly'); input = 'commit.c3dc67d0'; expected = 'soljson-v0.4.22-nightly.2018.4.12+commit.c3dc67d0.js'; - location = await provider.getVersionUrl(input, list); + location = await provider.getVersionUrlSegment(input, list); assert(location === expected, 'Should locate by commit'); input = '0.4.55.77-fantasy-solc'; expected = null; - location = await provider.getVersionUrl(input, list); + location = await provider.getVersionUrlSegment(input, list); assert(location === null, 'Should return null if not found') }); From 0b63bd0e474e0e14c1ab059d44f76b3f9caabc98 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Sun, 15 Apr 2018 16:08:57 -0700 Subject: [PATCH 07/22] Add babel for tests/ fix tests --- .travis.yml | 1 + compilerProvider.js | 3 +- package.json | 10 +++++- test/sources/ComplexOrdered.sol | 2 +- test/sources/MyContract.sol | 2 +- test/sources/NewPragma.sol | 2 +- test/sources/OldPragmaFloat.sol | 2 +- test/sources/OldPragmaPin.sol | 2 +- test/test_ordering.js | 15 ++++++--- test/test_provider.js | 57 +++++++++++++++++---------------- 10 files changed, 55 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index a994a80..5144f46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,4 +22,5 @@ script: -e TRAVIS_PULL_REQUEST_BRANCH \ -e TRAVIS_BRANCH \ -e TEST \ + -e CI \ truffle/ci:latest run_tests diff --git a/compilerProvider.js b/compilerProvider.js index fe94318..7e5a3f2 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -127,9 +127,8 @@ CompilerProvider.prototype.getLocal = function(localPath){ */ CompilerProvider.prototype.getVersions = function(){ const self = this; - url = self.config.versionsUrl; - return request(url) + return request(self.config.versionsUrl) .then(list => JSON.parse(list)) .catch(err => self.errors('noRequest', url, err)); } diff --git a/package.json b/package.json index 1d9fc44..bba63a4 100644 --- a/package.json +++ b/package.json @@ -18,12 +18,20 @@ "truffle-expect": "^0.0.3" }, "devDependencies": { + "babel-core": "^6.26.0", + "babel-polyfill": "^6.26.0", + "babel-preset-env": "^1.6.1", "mocha": "^3.5.3", "tmp": "0.0.33", "truffle-resolver": "2.0.0" }, + "babel": { + "presets": [ + "env" + ] + }, "scripts": { - "test": "mocha" + "test": "mocha --timeout 5000 --compilers js:babel-core/register --require babel-polyfill test" }, "repository": { "type": "git", diff --git a/test/sources/ComplexOrdered.sol b/test/sources/ComplexOrdered.sol index 84e51d3..460e240 100644 --- a/test/sources/ComplexOrdered.sol +++ b/test/sources/ComplexOrdered.sol @@ -16,4 +16,4 @@ contract ComplexOrdered is InheritA { function andThird() public pure {} } -contract Empty {} \ No newline at end of file +contract Empty {} diff --git a/test/sources/MyContract.sol b/test/sources/MyContract.sol index 05a5f91..0b67f68 100644 --- a/test/sources/MyContract.sol +++ b/test/sources/MyContract.sol @@ -15,4 +15,4 @@ library SomeLibrary { interface SomeInterface { -} \ No newline at end of file +} diff --git a/test/sources/NewPragma.sol b/test/sources/NewPragma.sol index 82916e5..0e75f70 100644 --- a/test/sources/NewPragma.sol +++ b/test/sources/NewPragma.sol @@ -6,4 +6,4 @@ contract NewPragma { function NewPragma() public { x = 5; } -} \ No newline at end of file +} diff --git a/test/sources/OldPragmaFloat.sol b/test/sources/OldPragmaFloat.sol index d505811..7aadf72 100644 --- a/test/sources/OldPragmaFloat.sol +++ b/test/sources/OldPragmaFloat.sol @@ -6,4 +6,4 @@ contract OldPragmaFloat { function OldPragmaFloat() public { x = 7; } -} \ No newline at end of file +} diff --git a/test/sources/OldPragmaPin.sol b/test/sources/OldPragmaPin.sol index e3c8fa3..04d28fb 100644 --- a/test/sources/OldPragmaPin.sol +++ b/test/sources/OldPragmaPin.sol @@ -6,4 +6,4 @@ contract OldPragmaPin { function OldPragmaPin() public { x = 7; } -} \ No newline at end of file +} diff --git a/test/test_ordering.js b/test/test_ordering.js index c1b3fd3..155a0c1 100644 --- a/test/test_ordering.js +++ b/test/test_ordering.js @@ -6,8 +6,9 @@ var assert = require("assert"); describe("Compile", function() { this.timeout(5000); // solc - var orderedSource = null; - var emptySource = null; + var simpleOrderedSource = null; + var complexOrderedSource = null; + var inheritedSource = null; var compileOptions = { contracts_directory: '', solc: ''}; describe("ABI Ordering", function(){ @@ -35,7 +36,7 @@ describe("Compile", function() { assert.deepEqual(abi, alphabetic); }); - it("orders the simple ABI", function(){ + it("orders the simple ABI", function(done){ var expectedOrder = ['theFirst', 'second', 'andThird']; var sources = {}; sources["SimpleOrdered.sol"] = simpleOrderedSource; @@ -45,6 +46,7 @@ describe("Compile", function() { return item.name; }); assert.deepEqual(abi, expectedOrder); + done(); }) }); @@ -67,7 +69,7 @@ describe("Compile", function() { assert.deepEqual(abi, alphabetic); }); - it("orders the complex ABI", function(){ + it("orders the complex ABI", function(done){ var expectedOrder = ['LogB', 'LogA', 'LogD', 'LogC', 'theFirst', 'second', 'andThird']; var sources = {}; sources["ComplexOrdered.sol"] = complexOrderedSource; @@ -78,17 +80,20 @@ describe("Compile", function() { return item.name; }); assert.deepEqual(abi, expectedOrder); + done(); }) }); // Ported from `truffle-solidity-utils` - it("orders the ABI of a contract without functions", function(){ + it("orders the ABI of a contract without functions", function(done){ var sources = {}; + // ComplexOrdered.sol includes contract `Empty` sources["ComplexOrdered.sol"] = complexOrderedSource; sources["InheritB.sol"] = inheritedSource; Compile(sources, compileOptions, function(err, result){ assert.equal(result["Empty"].abi.length, 0); + done(); }) }) }) diff --git a/test/test_provider.js b/test/test_provider.js index f16c92f..b454937 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -7,7 +7,7 @@ const compile = require("../index"); const CompilerProvider = require('../compilerProvider'); function waitSecond() { - return new Promise((resolve, reject) => setTimeout(() => resolve(), 1250)); + return new Promise((resolve, reject) => setTimeout(() => resolve(), 5000)); } describe('CompilerProvider', function(){ @@ -28,26 +28,26 @@ describe('CompilerProvider', function(){ let input = '0.4.21'; let expected = 'soljson-v0.4.21+commit.dfe3193c.js'; - location = await provider.getVersionUrlSegment(input, list); - assert(location === expected, 'Should locate by version'); + let fileName = await provider.getVersionUrlSegment(input, list); + assert(fileName === expected, 'Should locate by version'); input = 'nightly.2018.4.12'; expected = 'soljson-v0.4.22-nightly.2018.4.12+commit.c3dc67d0.js'; - location = await provider.getVersionUrlSegment(input, list); - assert(location === expected, 'Should locate by nightly'); + fileName = await provider.getVersionUrlSegment(input, list); + assert(fileName === expected, 'Should locate by nightly'); input = 'commit.c3dc67d0'; expected = 'soljson-v0.4.22-nightly.2018.4.12+commit.c3dc67d0.js'; - location = await provider.getVersionUrlSegment(input, list); - assert(location === expected, 'Should locate by commit'); + fileName = await provider.getVersionUrlSegment(input, list); + assert(fileName === expected, 'Should locate by commit'); input = '0.4.55.77-fantasy-solc'; expected = null; - location = await provider.getVersionUrlSegment(input, list); - assert(location === null, 'Should return null if not found') + fileName = await provider.getVersionUrlSegment(input, list); + assert(fileName === null, 'Should return null if not found') }); it('getReleases: should return a `releases` object', async function(){ @@ -62,8 +62,9 @@ describe('CompilerProvider', function(){ describe('integration', function(){ this.timeout(40000); - let newPragmaSource; // floating at ^0.4.21 - let oldPragmaSource; // pinned at 0.4.15 + let newPragmaSource; // ^0.4.21 + let oldPragmaPinSource; // 0.4.15 + let oldPragmaFloatSource; // ^0.4.15 const options = { contracts_directory: '', @@ -78,7 +79,7 @@ describe('CompilerProvider', function(){ newPragmaSource = { "NewPragma.sol": newPragma}; oldPragmaPinSource = { "OldPragmaPin.sol": oldPragmaPin}; - oldPragmaFloatSource = { "OldPragmaPin.sol": oldPragmaFloat}; + oldPragmaFloatSource = { "OldPragmaFloat.sol": oldPragmaFloat}; }); it('compiles w/ default solc if no compiler specified (float)', function(done){ @@ -162,6 +163,9 @@ describe('CompilerProvider', function(){ tmp.dir((err, dir) => { if(err) return done(err); + let initialAccessTime; + let finalAccessTime; + const truffleCacheDir = dir + "/truffle/solc/cache/"; const expectedCache = truffleCacheDir + 'soljson-v0.4.21+commit.dfe3193c.js'; @@ -173,22 +177,28 @@ describe('CompilerProvider', function(){ // Run compiler, expecting solc to be downloaded and cached. compile(newPragmaSource, options, (err, result) => { - assert(err === null); - assert(fs.existsSync(expectedCache)); + if (err) return done(err); + assert(fs.existsSync(expectedCache), 'Should have cached compiler'); // Get cached solc access time - const initialAccessTime = fs.statSync(expectedCache).atime.getTime(); + initialAccessTime = fs.statSync(expectedCache).atime.getTime() // Wait a second and recompile, verifying that the cached solc // got accessed / ran ok. waitSecond().then(() => { compile(newPragmaSource, options, (err, result) => { - const finalAccessTime = fs.statSync(expectedCache).atime.getTime(); + if (err) return done(err); + + finalAccessTime = fs.statSync(expectedCache).atime.getTime() + + assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); + + // atime is not getting updated on read in CI. + if (!process.env.CI){ + assert(initialAccessTime < finalAccessTime, "Should have used cached compiler"); + } - assert(err == null); - assert(result['NewPragma'].contract_name === 'NewPragma'); - assert(initialAccessTime < finalAccessTime); done(); }); @@ -198,12 +208,3 @@ describe('CompilerProvider', function(){ }); }); }); - - - - - - - - - From e3e252be5be1920b77b2c78056af61416a37bbbf Mon Sep 17 00:00:00 2001 From: cgewecke Date: Mon, 16 Apr 2018 16:42:23 -0700 Subject: [PATCH 08/22] Use original require, clarify code --- compilerProvider.js | 46 +++++++++++++++++++++++++++++-------------- package.json | 3 ++- test/test_provider.js | 19 +++++++++++------- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index 7e5a3f2..b1737ad 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -2,7 +2,8 @@ const path = require('path'); const fs = require('fs'); const request = require('request-promise'); const requireFromString = require('require-from-string'); -const mkdirp = require('mkdirp'); +const findCacheDir = require('find-cache-dir'); +const originalRequire = require('original-require'); //------------------------------ Constructor/Config ------------------------------------------------ @@ -23,12 +24,17 @@ function CompilerProvider(_config){ CompilerProvider.prototype.config = { solc: null, versionsUrl: 'https://solc-bin.ethereum.org/bin/list.json', - compilerUrl: 'https://solc-bin.ethereum.org/bin/', - compilerNpm: 'solc', + compilerUrlRoot: 'https://solc-bin.ethereum.org/bin/', cache: true, - cachePath: '/var/lib/truffle/cache/solc/', } + +CompilerProvider.prototype.cachePath = findCacheDir({ + name: 'truffle', + cwd: __dirname, + create: true, +}) + //----------------------------------- Interface --------------------------------------------------- /** @@ -52,7 +58,7 @@ CompilerProvider.prototype.load = function(options){ if (useDefault) return accept(self.getDefault()); if (useLocal) return accept(self.getLocal(solc)); - if (useRemote) return accept(self.getByUrl(solc)); + if (useRemote) return accept(self.getByUrl(solc)); // Tries cache first, then remote. }); } @@ -94,7 +100,7 @@ CompilerProvider.prototype.getReleases = function(){ * @return {Module} solc */ CompilerProvider.prototype.getDefault = function(){ - const compiler = require(this.config.compilerNpm); + const compiler = require('solc'); this.removeListener(); return compiler; } @@ -109,7 +115,7 @@ CompilerProvider.prototype.getLocal = function(localPath){ let compiler; try { - compiler = require(localPath) + compiler = originalRequire(localPath) self.removeListener(); } catch (err) { throw self.errors('noPath', localPath); @@ -176,7 +182,7 @@ CompilerProvider.prototype.getByUrl = function(version){ if (self.isCached(file)) return self.getFromCache(file); - const url = self.config.compilerUrl + file; + const url = self.config.compilerUrlRoot + file; return request .get(url) @@ -199,15 +205,27 @@ CompilerProvider.prototype.isLocal = function(localPath){ return fs.existsSync(localPath) || path.isAbsolute(localPath); } +/** + * Returns path to cached solc version + * @param {String} fileName ex: "soljson-v0.4.21+commit.dfe3193c.js" + * @return {String} path + */ +CompilerProvider.prototype.resolveCache = function(fileName){ + const thunk = findCacheDir({name: 'truffle', thunk: true}); + return thunk(fileName); +} + /** * Returns true if `fileName` exists in the cache. * @param {String} fileName ex: "soljson-v0.4.21+commit.dfe3193c.js" * @return {Boolean} */ CompilerProvider.prototype.isCached = function(fileName){ - return fs.existsSync(this.config.cachePath + fileName); + const file = this.resolveCache(fileName); + return fs.existsSync(file); } + /** * Write to the cache at `config.cachePath`. Creates `cachePath` directory if * does not exist. @@ -217,11 +235,8 @@ CompilerProvider.prototype.isCached = function(fileName){ CompilerProvider.prototype.addToCache = function(code, fileName){ if (!this.config.cache) return; - if (!fs.existsSync(this.config.cachePath)){ - mkdirp.sync(this.config.cachePath); - } - - fs.writeFileSync(this.config.cachePath + fileName, code); + const filePath = this.resolveCache(fileName); + fs.writeFileSync(filePath, code); } /** @@ -230,7 +245,8 @@ CompilerProvider.prototype.addToCache = function(code, fileName){ * @return {Module} solc */ CompilerProvider.prototype.getFromCache = function(fileName){ - const cached = fs.readFileSync(this.config.cachePath + fileName, "utf-8"); + const filePath = this.resolveCache(fileName) + const cached = fs.readFileSync(filePath, "utf-8"); return this.compilerFromString(cached); } diff --git a/package.json b/package.json index bba63a4..90d869a 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "async": "^2.1.4", "colors": "^1.1.2", "debug": "^3.1.0", - "mkdirp": "^0.5.1", + "find-cache-dir": "^1.0.0", + "original-require": "^1.0.1", "request": "^2.85.0", "request-promise": "^4.2.2", "require-from-string": "^2.0.2", diff --git a/test/test_provider.js b/test/test_provider.js index b454937..34b9745 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -3,6 +3,7 @@ const tmp = require("tmp"); const path = require("path"); const solc = require("solc"); const assert = require("assert"); +const findCacheDir = require('find-cache-dir'); const compile = require("../index"); const CompilerProvider = require('../compilerProvider'); @@ -86,7 +87,8 @@ describe('CompilerProvider', function(){ options.compiler = { cache: false }; compile(newPragmaSource, options, (err, result) => { - assert(err == null); + if (err) return done(err); + assert(result['NewPragma'].contract_name === 'NewPragma'); done(); }); @@ -99,7 +101,8 @@ describe('CompilerProvider', function(){ }; compile(oldPragmaPinSource, options, (err, result) => { - assert(err == null); + if (err) return done(err); + assert(result['OldPragmaPin'].contract_name === 'OldPragmaPin'); done(); }); @@ -113,7 +116,8 @@ describe('CompilerProvider', function(){ }; compile(oldPragmaFloatSource, options, (err, result) => { - assert(err == null); + if (err) return done(err); + assert(result['OldPragmaFloat'].contract_name === 'OldPragmaFloat'); done(); }); @@ -140,7 +144,8 @@ describe('CompilerProvider', function(){ }; compile(newPragmaSource, options, (err, result) => { - assert(err == null); + if (err) return done(err); + assert(result['NewPragma'].contract_name === 'NewPragma'); done(); }); @@ -166,18 +171,18 @@ describe('CompilerProvider', function(){ let initialAccessTime; let finalAccessTime; - const truffleCacheDir = dir + "/truffle/solc/cache/"; - const expectedCache = truffleCacheDir + 'soljson-v0.4.21+commit.dfe3193c.js'; + const thunk = findCacheDir({name: 'truffle', thunk: true}); + const expectedCache = thunk('soljson-v0.4.21+commit.dfe3193c.js'); options.compiler = { cache: true, - cachePath: truffleCacheDir, solc: "0.4.21" }; // Run compiler, expecting solc to be downloaded and cached. compile(newPragmaSource, options, (err, result) => { if (err) return done(err); + assert(fs.existsSync(expectedCache), 'Should have cached compiler'); // Get cached solc access time From dc8fd764122e281df9fdb88a7be73931a8cf1937 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Mon, 16 Apr 2018 17:29:15 -0700 Subject: [PATCH 09/22] Remove old public command contract name logic --- parser.js | 89 --------------------------------------------- profiler.js | 55 ---------------------------- test/test_parser.js | 27 -------------- 3 files changed, 171 deletions(-) diff --git a/parser.js b/parser.js index 383da84..b73708b 100644 --- a/parser.js +++ b/parser.js @@ -2,99 +2,10 @@ var CompileError = require("./compileerror"); var fs = require("fs"); var path = require("path"); -// Clean up after solc. -var listeners = process.listeners("uncaughtException"); -var solc_listener = listeners[listeners.length - 1]; - -if (solc_listener) { - process.removeListener("uncaughtException", solc_listener); -} - // Warning issued by a pre-release compiler version, ignored by this component. var preReleaseCompilerWarning = "This is a pre-release compiler version, please do not use it in production."; -var installedContractsDir = "installed_contracts" - module.exports = { - parse: function(body, fileName) { - // Here, we want a valid AST even if imports don't exist. The way to - // get around that is to tell the compiler, as they happen, that we - // have source for them (an empty file). - - var build_remappings = function() { - // Maps import paths to paths from EthPM installed contracts, so we can correctly solve imports - // e.g. "my_pkg/=installed_contracts/my_pkg/contracts/" - var remappings = []; - - if (fs.existsSync('ethpm.json')) { - ethpm = JSON.parse(fs.readFileSync('ethpm.json')); - for (pkg in ethpm.dependencies) { - remappings.push(pkg + "/=" + path.join(installedContractsDir, pkg, 'contracts', '/')); - } - } - - return remappings; - } - - var fileName = fileName || "ParsedContract.sol"; - - var remappings = build_remappings(); - - var solcStandardInput = { - language: "Solidity", - sources: { - [fileName]: { - content: body - } - }, - settings: { - remappings: remappings, - outputSelection: { - "*": { - "": [ - "ast" - ] - } - } - } - }; - - var output = solc.compileStandard(JSON.stringify(solcStandardInput), function(file_path) { - // Resolve dependency manually. - if (fs.existsSync(file_path)) { - contents = fs.readFileSync(file_path, {encoding: 'UTF-8'}); - } - else { - contents = "pragma solidity ^0.4.0;"; - } - return {contents: contents}; - }); - - output = JSON.parse(output); - - // Filter out the "pre-release compiler" warning, if present. - var errors = output.errors ? output.errors.filter(function(solidity_error) { - return solidity_error.message.indexOf(preReleaseCompilerWarning) < 0; - }) : []; - - // Filter out warnings. - var warnings = output.errors ? output.errors.filter(function(solidity_error) { - return solidity_error.severity == "warning"; - }) : []; - var errors = output.errors ? output.errors.filter(function(solidity_error) { - return solidity_error.severity != "warning"; - }) : []; - - if (errors.length > 0) { - throw new CompileError(errors[0].formattedMessage); - } - - return { - contracts: Object.keys(output.contracts[fileName]), - ast: output.sources[fileName].ast - }; - }, - // This needs to be fast! It is fast (as of this writing). Keep it fast! parseImports: function(body, solc) { var self = this; diff --git a/profiler.js b/profiler.js index a93cbd0..8e068f8 100644 --- a/profiler.js +++ b/profiler.js @@ -348,59 +348,4 @@ module.exports = { isExplicitlyRelative: function(import_path) { return import_path.indexOf(".") == 0; }, - - // Parse all source files in the directory and output the names of contracts and their source paths - // directory can either be a directory or array of files. - defined_contracts: function(directory, callback) { - function getFiles(callback) { - if (Array.isArray(directory)) { - callback(null, directory); - } else { - find_contracts(directory, callback); - } - } - - getFiles(function(err, files) { - if (err) return callback(err); - - var promises = files.map(function(file) { - return new Promise(function(accept, reject) { - fs.readFile(file, "utf8", function(err, body) { - if (err) return reject(err); - - var output; - - try { - output = Parser.parse(body); - } catch (e) { - e.message = "Error parsing " + file + ": " + e.message; - return reject(e); - } - - accept(output.contracts); - }); - }).then(function(contract_names) { - var returnVal = {}; - - contract_names.forEach(function(contract_name) { - returnVal[contract_name] = file; - }); - - return returnVal; - }); - }); - - Promise.all(promises).then(function(objects) { - var contract_source_paths = {}; - - objects.forEach(function(object) { - Object.keys(object).forEach(function(contract_name) { - contract_source_paths[contract_name] = object[contract_name]; - }); - }); - - callback(null, contract_source_paths); - }).catch(callback); - }); - } }; diff --git a/test/test_parser.js b/test/test_parser.js index 8c9301e..801d8a1 100644 --- a/test/test_parser.js +++ b/test/test_parser.js @@ -47,31 +47,4 @@ describe("Parser", function() { assert(error.message.indexOf("Expected pragma, import directive or contract") >= 0); }); - - it.skip("should return a full AST when parsed, even when dependencies don't exist", function() { - this.timeout(4000); - - var output = Parser.parse(source); - - assert.deepEqual(output.contracts, ["MyContract", "SomeInterface", "SomeLibrary"]); - assert(output.ast.nodes.length > 0); - - // The above assert means we at least got some kind of AST. - // Is there something we specifically need here? - }); - - it.skip("should throw an error when parsing completely if there's an actual parse error", function() { - var error = null; - try { - Parser.parse(erroneousSource); - } catch(e) { - error = e; - } - - if (!error) { - throw new Error("Expected a parse error but didn't get one!"); - } - - assert(error.message.indexOf("Expected pragma, import directive or contract") >= 0); - }); }); From fcd4a0d5e70449443a6e4ed5b121afb3d4446264 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Tue, 17 Apr 2018 13:20:43 -0700 Subject: [PATCH 10/22] Specify cwd on resolve & throw error correctly --- compilerProvider.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index b1737ad..f65f446 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -190,7 +190,7 @@ CompilerProvider.prototype.getByUrl = function(version){ self.addToCache(response, file); return self.compilerFromString(response); }) - .catch(err => self.errors('noRequest', url, err)); + .catch(err => { throw self.errors('noRequest', url, err)}); }); } @@ -211,7 +211,7 @@ CompilerProvider.prototype.isLocal = function(localPath){ * @return {String} path */ CompilerProvider.prototype.resolveCache = function(fileName){ - const thunk = findCacheDir({name: 'truffle', thunk: true}); + const thunk = findCacheDir({name: 'truffle', cwd: __dirname, thunk: true}); return thunk(fileName); } From 5877c25f9335dad574a9bd040e737b72544929ae Mon Sep 17 00:00:00 2001 From: cgewecke Date: Tue, 17 Apr 2018 16:35:58 -0700 Subject: [PATCH 11/22] Add exception handler for remote requires --- compilerProvider.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compilerProvider.js b/compilerProvider.js index f65f446..1c4b1da 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -258,7 +258,9 @@ CompilerProvider.prototype.getFromCache = function(fileName){ CompilerProvider.prototype.compilerFromString = function(code){ const solc = this.getDefault(); const compiler = requireFromString(code); - return solc.setupMethods(compiler); + const wrapped = solc.setupMethods(compiler); + this.removeListener(); + return wrapped; } /** From e21bb3acd4502934a0b2a96b57535a80f85a6d63 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 18 Apr 2018 12:04:40 -0700 Subject: [PATCH 12/22] Add dockerized solc --- compilerProvider.js | 88 ++++++++++++++++++++++++++++++++++----- package.json | 1 - test/test_provider.js | 97 +++++++++++++++++++++++++++++-------------- 3 files changed, 142 insertions(+), 44 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index 1c4b1da..f595c45 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -1,5 +1,6 @@ const path = require('path'); const fs = require('fs'); +const child = require('child_process'); const request = require('request-promise'); const requireFromString = require('require-from-string'); const findCacheDir = require('find-cache-dir'); @@ -38,24 +39,28 @@ CompilerProvider.prototype.cachePath = findCacheDir({ //----------------------------------- Interface --------------------------------------------------- /** - * Loads solc from four possible locations: - * - local node_modules (param: ) - * - absolute path to a local solc (param: ) - * - a solc cache (param: && cache contains version) - * - a remote solc-bin source (param: && version not cached) + * Load solcjs from four possible locations: + * - local node_modules (config.solc = ) + * - absolute path to a local solc (config.solc = ) + * - a solc cache (config.solc = && cache contains version) + * - a remote solc-bin source (config.solc = && version not cached) * - * @param {String} options [optional] options to pass to native solc binary - * @return {Module} solc + * OR specify that solc.compileStandard should wrap: + * - dockerized solc (config.solc = "" && config.docker: true) + * + * @return {Module|Object} solc */ -CompilerProvider.prototype.load = function(options){ +CompilerProvider.prototype.load = function(){ const self = this; const solc = self.config.solc; return new Promise((accept, reject) => { + const useDocker = self.config.docker; const useDefault = !solc; const useLocal = !useDefault && self.isLocal(solc); - const useRemote = !useLocal; + const useRemote = !useLocal + if (useDocker) return accept(self.getDocker()); if (useDefault) return accept(self.getDefault()); if (useLocal) return accept(self.getLocal(solc)); if (useRemote) return accept(self.getByUrl(solc)); // Tries cache first, then remote. @@ -124,6 +129,7 @@ CompilerProvider.prototype.getLocal = function(localPath){ return compiler; } + /** * Fetches solc versions object from remote solc-bin. This includes an array of build * objects with detailed version info, an array of release version numbers @@ -136,7 +142,7 @@ CompilerProvider.prototype.getVersions = function(){ return request(self.config.versionsUrl) .then(list => JSON.parse(list)) - .catch(err => self.errors('noRequest', url, err)); + .catch(err => {throw self.errors('noRequest', url, err)}); } /** @@ -194,6 +200,36 @@ CompilerProvider.prototype.getByUrl = function(version){ }); } +/** + * Makes solc.compileStandard a wrapper to a child process invocation of dockerized solc. + * @return {Object} solc output + */ +CompilerProvider.prototype.getDocker = function(){ + const version = this.config.solc; + + this.validateDocker(); + + return this + .getVersions() + .then(versions => { + return { + + compileStandard: (options) => { + const command = 'docker run -i ethereum/solc:' + version + ' --standard-json'; + + const result = child.execSync(command, {input: options}); + return String(result); + }, + + version: () => { + return (version === 'stable') + ? versions.latestRelease + : version + } + } + }) +} + //------------------------------------ Utils ------------------------------------------------------- /** @@ -205,6 +241,34 @@ CompilerProvider.prototype.isLocal = function(localPath){ return fs.existsSync(localPath) || path.isAbsolute(localPath); } +/** + * Checks to make sure image is specified in the config, that docker exists and that + * the image exists locally. If the last condition isn't true, docker will try to pull + * it down and this breaks everything. + * @throws {Error} + */ +CompilerProvider.prototype.validateDocker = function(){ + let result; + const image = this.config.solc; + + // Image specified + if (!image) throw this.errors('noString', image); + + // Docker exists locally + try { + result = child.execSync('docker -v'); + } catch(err){ + throw this.errors('noDocker'); + } + + // Image exists locally + try { + child.execSync('docker inspect --type=image ethereum/solc:' + image); + } catch(err){ + throw this.errors('noImage', image); + } +} + /** * Returns path to cached solc version * @param {String} fileName ex: "soljson-v0.4.21+commit.dfe3193c.js" @@ -289,8 +353,10 @@ CompilerProvider.prototype.errors = function(kind, input, err){ const kinds = { noPath: "Could not find compiler at: " + input, noVersion: "Could not find compiler version:\n" + input + ".\n" + info, - noString: "Expected string (`path` or `version`), got: " + input.toString(), + noString: "`compiler.solc` option must be string specifying a path, solc version, or docker image, got: " + input, noRequest: "Failed to complete request to: " + input + ".\n\n" + err, + noDocker: "You are trying to run dockerized solc, but docker is not installed on this machine.", + noImage: "Please pull " + input + " from docker before trying to compile with it.", } return new Error(kinds[kind]); diff --git a/package.json b/package.json index 09997f4..36bd543 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "babel-polyfill": "^6.26.0", "babel-preset-env": "^1.6.1", "mocha": "^3.5.3", - "tmp": "0.0.33", "truffle-resolver": "2.0.0" }, "babel": { diff --git a/test/test_provider.js b/test/test_provider.js index 34b9745..55a79ad 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -1,5 +1,4 @@ const fs = require("fs"); -const tmp = require("tmp"); const path = require("path"); const solc = require("solc"); const assert = require("assert"); @@ -165,51 +164,85 @@ describe('CompilerProvider', function(){ }); it('caches releases and uses them if available', function(done){ - tmp.dir((err, dir) => { - if(err) return done(err); + let initialAccessTime; + let finalAccessTime; - let initialAccessTime; - let finalAccessTime; + const thunk = findCacheDir({name: 'truffle', thunk: true}); + const expectedCache = thunk('soljson-v0.4.21+commit.dfe3193c.js'); - const thunk = findCacheDir({name: 'truffle', thunk: true}); - const expectedCache = thunk('soljson-v0.4.21+commit.dfe3193c.js'); + options.compiler = { + cache: true, + solc: "0.4.21" + }; - options.compiler = { - cache: true, - solc: "0.4.21" - }; + // Run compiler, expecting solc to be downloaded and cached. + compile(newPragmaSource, options, (err, result) => { + if (err) return done(err); - // Run compiler, expecting solc to be downloaded and cached. - compile(newPragmaSource, options, (err, result) => { - if (err) return done(err); + assert(fs.existsSync(expectedCache), 'Should have cached compiler'); - assert(fs.existsSync(expectedCache), 'Should have cached compiler'); + // Get cached solc access time + initialAccessTime = fs.statSync(expectedCache).atime.getTime() - // Get cached solc access time - initialAccessTime = fs.statSync(expectedCache).atime.getTime() + // Wait a second and recompile, verifying that the cached solc + // got accessed / ran ok. + waitSecond().then(() => { - // Wait a second and recompile, verifying that the cached solc - // got accessed / ran ok. - waitSecond().then(() => { + compile(newPragmaSource, options, (err, result) => { + if (err) return done(err); - compile(newPragmaSource, options, (err, result) => { - if (err) return done(err); + finalAccessTime = fs.statSync(expectedCache).atime.getTime() - finalAccessTime = fs.statSync(expectedCache).atime.getTime() + assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); - assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); + // atime is not getting updated on read in CI. + if (!process.env.CI){ + assert(initialAccessTime < finalAccessTime, "Should have used cached compiler"); + } - // atime is not getting updated on read in CI. - if (!process.env.CI){ - assert(initialAccessTime < finalAccessTime, "Should have used cached compiler"); - } + done(); + }); - done(); - }); + }).catch(done); + }); + }); - }).catch(done); - }); + it('compiles with dockerized solc', function(done){ + options.compiler = { + solc: "0.4.22", + docker: true + }; + + compile(newPragmaSource, options, (err, result) => { + assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); + done(); }); }); + + it('errors if running dockerized solc without specifying an image', function(done){ + options.compiler = { + solc: undefined, + docker: true + }; + + compile(newPragmaSource, options, (err, result) => { + assert(err.message.includes('option must be')); + done(); + }); + }) + + it('errors if running dockerized solc when image does not exist locally', function(done){ + const imageName = 'fantasySolc.777555'; + + options.compiler = { + solc: imageName, + docker: true + }; + + compile(newPragmaSource, options, (err, result) => { + assert(err.message.includes(imageName)); + done(); + }); + }) }); }); From 77428dfc261ad2cfcb95d2ece182803e5276aaf7 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 18 Apr 2018 13:59:19 -0700 Subject: [PATCH 13/22] Run dockerized solc in CI --- .travis.yml | 18 ++++------- compilerProvider.js | 1 - package.json | 4 ++- scripts/ci.sh | 26 ++++++++++++++++ test/test_provider.js | 72 ++++++++++++++++++++++--------------------- 5 files changed, 72 insertions(+), 49 deletions(-) create mode 100755 scripts/ci.sh diff --git a/.travis.yml b/.travis.yml index 5144f46..a0eabb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ +dist: trusty sudo: required - -language: generic +language: node_js +node_js: + - "6" services: - docker @@ -9,18 +11,10 @@ before_install: - docker pull truffle/ci env: + - TEST=native - TEST=repo - TEST=upstream - TEST=scenario script: - - > - docker run -it --rm --name ${TEST} \ - -e TRAVIS_REPO_SLUG \ - -e TRAVIS_PULL_REQUEST \ - -e TRAVIS_PULL_REQUEST_SLUG \ - -e TRAVIS_PULL_REQUEST_BRANCH \ - -e TRAVIS_BRANCH \ - -e TEST \ - -e CI \ - truffle/ci:latest run_tests + - npm run test:ci diff --git a/compilerProvider.js b/compilerProvider.js index f595c45..3632241 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -289,7 +289,6 @@ CompilerProvider.prototype.isCached = function(fileName){ return fs.existsSync(file); } - /** * Write to the cache at `config.cachePath`. Creates `cachePath` directory if * does not exist. diff --git a/package.json b/package.json index 36bd543..e3f0efc 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,9 @@ ] }, "scripts": { - "test": "mocha --timeout 5000 --compilers js:babel-core/register --require babel-polyfill test" + "test": "mocha --invert --grep native --timeout 5000 --compilers js:babel-core/register --require babel-polyfill test", + "test:native": "mocha --grep native --timeout 5000 --compilers js:babel-core/register --require babel-polyfill test", + "test:ci": "./scripts/ci.sh" }, "repository": { "type": "git", diff --git a/scripts/ci.sh b/scripts/ci.sh new file mode 100755 index 0000000..a9b61a1 --- /dev/null +++ b/scripts/ci.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -o errexit + +run_native_tests() { + docker pull ethereum/solc:0.4.22 + npm install + npm run test:native +} + +run_container_tests() { + docker run -it --rm --name ${TEST} \ + -e TRAVIS_REPO_SLUG \ + -e TRAVIS_PULL_REQUEST \ + -e TRAVIS_PULL_REQUEST_SLUG \ + -e TRAVIS_PULL_REQUEST_BRANCH \ + -e TRAVIS_BRANCH \ + -e TEST \ + truffle/ci:latest run_tests +} + +if [ "$TEST" == "native" ]; then + run_solc_native_tests +else + run_container_tests +fi diff --git a/test/test_provider.js b/test/test_provider.js index 55a79ad..f045971 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -196,53 +196,55 @@ describe('CompilerProvider', function(){ assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); // atime is not getting updated on read in CI. - if (!process.env.CI){ + if (!process.env.TEST){ assert(initialAccessTime < finalAccessTime, "Should have used cached compiler"); } done(); }); - }).catch(done); }); }); - it('compiles with dockerized solc', function(done){ - options.compiler = { - solc: "0.4.22", - docker: true - }; - - compile(newPragmaSource, options, (err, result) => { - assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); - done(); - }); - }); + describe('docker [ @native ]', function() { - it('errors if running dockerized solc without specifying an image', function(done){ - options.compiler = { - solc: undefined, - docker: true - }; + it('compiles with dockerized solc', function(done){ + options.compiler = { + solc: "0.4.22", + docker: true + }; - compile(newPragmaSource, options, (err, result) => { - assert(err.message.includes('option must be')); - done(); + compile(newPragmaSource, options, (err, result) => { + assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); + done(); + }); }); - }) - - it('errors if running dockerized solc when image does not exist locally', function(done){ - const imageName = 'fantasySolc.777555'; - - options.compiler = { - solc: imageName, - docker: true - }; - compile(newPragmaSource, options, (err, result) => { - assert(err.message.includes(imageName)); - done(); - }); - }) + it('errors if running dockerized solc without specifying an image', function(done){ + options.compiler = { + solc: undefined, + docker: true + }; + + compile(newPragmaSource, options, (err, result) => { + assert(err.message.includes('option must be')); + done(); + }); + }) + + it('errors if running dockerized solc when image does not exist locally', function(done){ + const imageName = 'fantasySolc.777555'; + + options.compiler = { + solc: imageName, + docker: true + }; + + compile(newPragmaSource, options, (err, result) => { + assert(err.message.includes(imageName)); + done(); + }); + }) + }); }); }); From 84b1dcc7aad555823848c9a655bfdb328961e78c Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 18 Apr 2018 14:33:59 -0700 Subject: [PATCH 14/22] Add native compilation --- compilerProvider.js | 57 +++++++++++++++++++++++++++++++------------ scripts/ci.sh | 6 ++++- test/test_provider.js | 15 ++++++++++-- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index 3632241..25e47a2 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -53,16 +53,19 @@ CompilerProvider.prototype.cachePath = findCacheDir({ CompilerProvider.prototype.load = function(){ const self = this; const solc = self.config.solc; + const isNative = self.config.solc === 'native'; return new Promise((accept, reject) => { const useDocker = self.config.docker; const useDefault = !solc; const useLocal = !useDefault && self.isLocal(solc); - const useRemote = !useLocal + const useNative = !useLocal && isNative; + const useRemote = !useNative if (useDocker) return accept(self.getDocker()); if (useDefault) return accept(self.getDefault()); if (useLocal) return accept(self.getLocal(solc)); + if (useNative) return accept(self.getNative(solc)); if (useRemote) return accept(self.getByUrl(solc)); // Tries cache first, then remote. }); } @@ -209,25 +212,47 @@ CompilerProvider.prototype.getDocker = function(){ this.validateDocker(); - return this - .getVersions() - .then(versions => { - return { + return this.getVersions().then(versions => { + return { - compileStandard: (options) => { - const command = 'docker run -i ethereum/solc:' + version + ' --standard-json'; + compileStandard: (options) => { + const command = 'docker run -i ethereum/solc:' + version + ' --standard-json'; + const result = child.execSync(command, {input: options}); + return String(result); + }, - const result = child.execSync(command, {input: options}); - return String(result); - }, + version: () => { + return (version === 'stable') + ? versions.latestRelease + : version + } + } + }) +} - version: () => { - return (version === 'stable') - ? versions.latestRelease - : version - } +/** + * Makes solc.compileStandard a wrapper to a child process invocation of dockerized solc. + * @return {Object} solc output + */ +CompilerProvider.prototype.getNative = function(){ + const version = this.config.solc; + + return this.getVersions().then(versions => { + return { + + compileStandard: (options) => { + const command = 'solc --standard-json'; + const result = child.execSync(command, {input: options}); + return String(result); + }, + + version: () => { + return (version === 'stable') + ? versions.latestRelease + : version } - }) + } + }) } //------------------------------------ Utils ------------------------------------------------------- diff --git a/scripts/ci.sh b/scripts/ci.sh index a9b61a1..49ada75 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -3,6 +3,10 @@ set -o errexit run_native_tests() { + sudo add-apt-repository --yes ppa:ethereum/ethereum + sudo apt-get update + sudo apt-get install solc + docker pull ethereum/solc:0.4.22 npm install npm run test:native @@ -20,7 +24,7 @@ run_container_tests() { } if [ "$TEST" == "native" ]; then - run_solc_native_tests + run_native_tests else run_container_tests fi diff --git a/test/test_provider.js b/test/test_provider.js index f045971..fa2dcb4 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -206,7 +206,18 @@ describe('CompilerProvider', function(){ }); }); - describe('docker [ @native ]', function() { + describe('native / docker [ @native ]', function() { + + it('compiles with native solc', function(done){ + options.compiler = { + solc: "native" + }; + + compile(newPragmaSource, options, (err, result) => { + assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); + done(); + }); + }); it('compiles with dockerized solc', function(done){ options.compiler = { @@ -233,7 +244,7 @@ describe('CompilerProvider', function(){ }) it('errors if running dockerized solc when image does not exist locally', function(done){ - const imageName = 'fantasySolc.777555'; + const imageName = 'fantasySolc.7777555'; options.compiler = { solc: imageName, From fbe5bcc49a1f5d1d5587af3fac36ab4cf29aee34 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 18 Apr 2018 15:51:03 -0700 Subject: [PATCH 15/22] Cleanup errors/comments/tests --- compilerProvider.js | 14 +++++++++++--- test/test_provider.js | 4 ++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index 25e47a2..d74a299 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -47,6 +47,7 @@ CompilerProvider.prototype.cachePath = findCacheDir({ * * OR specify that solc.compileStandard should wrap: * - dockerized solc (config.solc = "" && config.docker: true) + * - native built solc (cofing.solc = "native") * * @return {Module|Object} solc */ @@ -375,12 +376,19 @@ CompilerProvider.prototype.errors = function(kind, input, err){ const info = 'Run `truffle compile --list` to see available versions.' const kinds = { + noPath: "Could not find compiler at: " + input, noVersion: "Could not find compiler version:\n" + input + ".\n" + info, - noString: "`compiler.solc` option must be string specifying a path, solc version, or docker image, got: " + input, noRequest: "Failed to complete request to: " + input + ".\n\n" + err, - noDocker: "You are trying to run dockerized solc, but docker is not installed on this machine.", - noImage: "Please pull " + input + " from docker before trying to compile with it.", + noDocker: "You are trying to run dockerized solc, but docker is not installed.", + noImage: "Please pull " + input + " from docker before trying to compile with it.", + + // Lists + noString: "`compiler.solc` option must be a string specifying:\n" + + " - a path to a locally installed solcjs\n" + + " - a solc version (ex: '0.4.22')\n" + + " - a docker image name (ex: 'stable')\n" + + "Received: " + input + " instead.", } return new Error(kinds[kind]); diff --git a/test/test_provider.js b/test/test_provider.js index fa2dcb4..6096e2d 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -214,6 +214,8 @@ describe('CompilerProvider', function(){ }; compile(newPragmaSource, options, (err, result) => { + if (err) return done(err); + assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); done(); }); @@ -226,6 +228,8 @@ describe('CompilerProvider', function(){ }; compile(newPragmaSource, options, (err, result) => { + if (err) return done(err); + assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); done(); }); From a06dc34f20cc0f101b4c2496b2bd2532e5ba0a7c Mon Sep 17 00:00:00 2001 From: cgewecke Date: Sat, 21 Apr 2018 15:33:59 -0700 Subject: [PATCH 16/22] Get correct version strings for docker/native --- compilerProvider.js | 88 ++++++++++++++++++++++++++----------------- index.js | 3 +- test/test_provider.js | 5 +++ 3 files changed, 61 insertions(+), 35 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index d74a299..b727a56 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -210,25 +210,17 @@ CompilerProvider.prototype.getByUrl = function(version){ */ CompilerProvider.prototype.getDocker = function(){ const version = this.config.solc; + const versionString = this.validateDocker(); - this.validateDocker(); + return { + compileStandard: (options) => { + const command = 'docker run -i ethereum/solc:' + version + ' --standard-json'; + const result = child.execSync(command, {input: options}); + return String(result); + }, - return this.getVersions().then(versions => { - return { - - compileStandard: (options) => { - const command = 'docker run -i ethereum/solc:' + version + ' --standard-json'; - const result = child.execSync(command, {input: options}); - return String(result); - }, - - version: () => { - return (version === 'stable') - ? versions.latestRelease - : version - } - } - }) + version: () => versionString, + } } /** @@ -237,23 +229,17 @@ CompilerProvider.prototype.getDocker = function(){ */ CompilerProvider.prototype.getNative = function(){ const version = this.config.solc; + const versionString = this.validateNative(); - return this.getVersions().then(versions => { - return { - - compileStandard: (options) => { - const command = 'solc --standard-json'; - const result = child.execSync(command, {input: options}); - return String(result); - }, + return { + compileStandard: (options) => { + const command = 'solc --standard-json'; + const result = child.execSync(command, {input: options}); + return String(result); + }, - version: () => { - return (version === 'stable') - ? versions.latestRelease - : version - } - } - }) + version: () => versionString, + } } //------------------------------------ Utils ------------------------------------------------------- @@ -271,10 +257,10 @@ CompilerProvider.prototype.isLocal = function(localPath){ * Checks to make sure image is specified in the config, that docker exists and that * the image exists locally. If the last condition isn't true, docker will try to pull * it down and this breaks everything. + * @return {String} solc version string * @throws {Error} */ CompilerProvider.prototype.validateDocker = function(){ - let result; const image = this.config.solc; // Image specified @@ -282,7 +268,7 @@ CompilerProvider.prototype.validateDocker = function(){ // Docker exists locally try { - result = child.execSync('docker -v'); + child.execSync('docker -v'); } catch(err){ throw this.errors('noDocker'); } @@ -293,6 +279,39 @@ CompilerProvider.prototype.validateDocker = function(){ } catch(err){ throw this.errors('noImage', image); } + + // Get version + const version = child.execSync('docker run ethereum/solc:' + image + ' --version'); + return this.normalizeVersion(version); +} + +/** + * Checks to make sure image is specified in the config, that docker exists and that + * the image exists locally. If the last condition isn't true, docker will try to pull + * it down and this breaks everything. + * @return {String} solc version string + * @throws {Error} + */ +CompilerProvider.prototype.validateNative = function(){ + let version; + try { + version = child.execSync('solc --version'); + } catch(err){ + throw this.errors('noNative', null, err); + } + + return this.normalizeVersion(version); +} + +/** + * Converts shell exec'd solc version from buffer to string and strips out human readable + * description. + * @param {Buffer} version result of childprocess + * @return {String} normalized version string: e.g 0.4.22+commit.4cb486ee.Linux.g++ + */ +CompilerProvider.prototype.normalizeVersion = function(version){ + version = String(version); + return version.split(':')[1].trim(); } /** @@ -382,6 +401,7 @@ CompilerProvider.prototype.errors = function(kind, input, err){ noRequest: "Failed to complete request to: " + input + ".\n\n" + err, noDocker: "You are trying to run dockerized solc, but docker is not installed.", noImage: "Please pull " + input + " from docker before trying to compile with it.", + noNative: "Could not execute local solc binary: " + err, // Lists noString: "`compiler.solc` option must be a string specifying:\n" + diff --git a/index.js b/index.js index 647b6c8..41fe1c1 100644 --- a/index.js +++ b/index.js @@ -387,6 +387,7 @@ compile.display = function(paths, options){ options.logger.log("Compiling " + contract + "..."); }); } -} +}; +compile.CompilerProvider = CompilerProvider; module.exports = compile; diff --git a/test/test_provider.js b/test/test_provider.js index 6096e2d..0d989f2 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -82,6 +82,7 @@ describe('CompilerProvider', function(){ oldPragmaFloatSource = { "OldPragmaFloat.sol": oldPragmaFloat}; }); + it('compiles w/ default solc if no compiler specified (float)', function(done){ options.compiler = { cache: false }; @@ -216,6 +217,7 @@ describe('CompilerProvider', function(){ compile(newPragmaSource, options, (err, result) => { if (err) return done(err); + assert(result['NewPragma'].compiler.version.included('Linux.g++')); assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); done(); }); @@ -227,9 +229,12 @@ describe('CompilerProvider', function(){ docker: true }; + const expectedVersion = '0.4.22+commit.4cb486ee.Linux.g++'; + compile(newPragmaSource, options, (err, result) => { if (err) return done(err); + assert(result['NewPragma'].compiler.version === expectedVersion); assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); done(); }); From 769c7b5ee97e9109d0980de76ff9282a0aa9555b Mon Sep 17 00:00:00 2001 From: cgewecke Date: Sun, 22 Apr 2018 13:09:54 -0700 Subject: [PATCH 17/22] Add method to list docker images --- compilerProvider.js | 20 ++++++++++++++++++++ test/test_provider.js | 8 +++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compilerProvider.js b/compilerProvider.js index b727a56..2105575 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -26,6 +26,7 @@ CompilerProvider.prototype.config = { solc: null, versionsUrl: 'https://solc-bin.ethereum.org/bin/list.json', compilerUrlRoot: 'https://solc-bin.ethereum.org/bin/', + dockerTagsUrl: 'https://registry.hub.docker.com/v2/repositories/ethereum/solc/tags/', cache: true, } @@ -102,6 +103,24 @@ CompilerProvider.prototype.getReleases = function(){ }); } +/** + * Fetches the first page of docker tags for the the ethereum/solc image + * @return {Object} tags + */ +CompilerProvider.prototype.getDockerTags = function(){ + const self = this; + + return request(self.config.dockerTagsUrl) + .then(list => + JSON + .parse(list) + .results + .map(item => item.name) + ) + .catch(err => {throw self.errors('noRequest', url, err)}); +} + + //------------------------------------ Getters ----------------------------------------------------- /** @@ -149,6 +168,7 @@ CompilerProvider.prototype.getVersions = function(){ .catch(err => {throw self.errors('noRequest', url, err)}); } + /** * Returns terminal url segment for `version` from the versions object * generated by `getVersions`. diff --git a/test/test_provider.js b/test/test_provider.js index 0d989f2..3942b33 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -58,6 +58,12 @@ describe('CompilerProvider', function(){ assert(releases.prereleases[0] === firstSolc, 'Should return prereleases'); assert(releases.releases[0] === releases.latestRelease, 'Should return releases/latestRelease'); }); + + it('lists available docker images [ @native ]', async function(){ + const list = await provider.getDockerTags(); + assert(Array.isArray(list)); + assert(typeof list[0] === 'string'); + }) }); describe('integration', function(){ @@ -217,7 +223,7 @@ describe('CompilerProvider', function(){ compile(newPragmaSource, options, (err, result) => { if (err) return done(err); - assert(result['NewPragma'].compiler.version.included('Linux.g++')); + assert(result['NewPragma'].compiler.version.includes('Linux.g++')); assert(result['NewPragma'].contract_name === 'NewPragma', 'Should have compiled'); done(); }); From 243f1fa86b2d35ff3104e8fd94be9ed0649345ec Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 25 Apr 2018 15:21:24 -0700 Subject: [PATCH 18/22] Parse imports correctly (built compilers) --- compilerProvider.js | 72 +++++++++++++------------ parser.js | 3 ++ test/{sources => mock}/MyContract.sol | 0 test/{sources => mock}/OldPragmaPin.sol | 0 test/{sources => mock}/ShouldError.sol | 0 test/test_ordering.js | 2 +- test/test_parser.js | 4 +- test/test_provider.js | 33 +++++++++++- 8 files changed, 77 insertions(+), 37 deletions(-) rename test/{sources => mock}/MyContract.sol (100%) rename test/{sources => mock}/OldPragmaPin.sol (100%) rename test/{sources => mock}/ShouldError.sol (100%) diff --git a/compilerProvider.js b/compilerProvider.js index 2105575..6dff396 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -64,10 +64,10 @@ CompilerProvider.prototype.load = function(){ const useNative = !useLocal && isNative; const useRemote = !useNative - if (useDocker) return accept(self.getDocker()); + if (useDocker) return accept(self.getBuilt("docker")); + if (useNative) return accept(self.getBuilt("native")); if (useDefault) return accept(self.getDefault()); if (useLocal) return accept(self.getLocal(solc)); - if (useNative) return accept(self.getNative(solc)); if (useRemote) return accept(self.getByUrl(solc)); // Tries cache first, then remote. }); } @@ -225,41 +225,36 @@ CompilerProvider.prototype.getByUrl = function(version){ } /** - * Makes solc.compileStandard a wrapper to a child process invocation of dockerized solc. + * Makes solc.compileStandard a wrapper to a child process invocation of dockerized solc + * or natively build solc. Also fetches a companion solcjs for the built js to parse imports * @return {Object} solc output */ -CompilerProvider.prototype.getDocker = function(){ - const version = this.config.solc; - const versionString = this.validateDocker(); - - return { - compileStandard: (options) => { - const command = 'docker run -i ethereum/solc:' + version + ' --standard-json'; - const result = child.execSync(command, {input: options}); - return String(result); - }, - - version: () => versionString, +CompilerProvider.prototype.getBuilt = function(buildType){ + let versionString; + let command; + + switch (buildType) { + case "native": + versionString = this.validateNative(); + command = 'solc --standard-json'; + break; + case "docker": + versionString = this.validateDocker(); + command = 'docker run -i ethereum/solc:' + this.config.solc + ' --standard-json'; + break; } -} -/** - * Makes solc.compileStandard a wrapper to a child process invocation of dockerized solc. - * @return {Object} solc output - */ -CompilerProvider.prototype.getNative = function(){ - const version = this.config.solc; - const versionString = this.validateNative(); - - return { - compileStandard: (options) => { - const command = 'solc --standard-json'; - const result = child.execSync(command, {input: options}); - return String(result); - }, - - version: () => versionString, - } + const commit = this.getCommitFromVersion(versionString); + + return this + .getByUrl(commit) + .then(solcjs => { + return { + compileStandard: (options) => String(child.execSync(command, {input: options})), + version: () => versionString, + importsParser: solcjs, + } + }); } //------------------------------------ Utils ------------------------------------------------------- @@ -323,6 +318,17 @@ CompilerProvider.prototype.validateNative = function(){ return this.normalizeVersion(version); } +/** + * Extracts a commit key from the version info returned by native/docker solc. + * We use this to fetch a companion solcjs from solc-bin in order to parse imports + * correctly. + * @param {String} versionString version info from ex: `solc -v` + * @return {String} commit key, ex: commit.4cb486ee + */ +CompilerProvider.prototype.getCommitFromVersion = function(versionString){ + return 'commit.' + versionString.match(/commit\.(.*?)\./)[1] +} + /** * Converts shell exec'd solc version from buffer to string and strips out human readable * description. diff --git a/parser.js b/parser.js index b73708b..42cddb6 100644 --- a/parser.js +++ b/parser.js @@ -19,6 +19,9 @@ module.exports = { // statement right on the end; just to ensure it will error and we can parse // the imports speedily without doing extra work. + // If we're using docker/native, we'll still want to use solcjs to do this part. + if (solc.importsParser) solc = solc.importsParser; + // Helper to detect import errors with an easy regex. var importErrorKey = "TRUFFLE_IMPORT"; diff --git a/test/sources/MyContract.sol b/test/mock/MyContract.sol similarity index 100% rename from test/sources/MyContract.sol rename to test/mock/MyContract.sol diff --git a/test/sources/OldPragmaPin.sol b/test/mock/OldPragmaPin.sol similarity index 100% rename from test/sources/OldPragmaPin.sol rename to test/mock/OldPragmaPin.sol diff --git a/test/sources/ShouldError.sol b/test/mock/ShouldError.sol similarity index 100% rename from test/sources/ShouldError.sol rename to test/mock/ShouldError.sol diff --git a/test/test_ordering.js b/test/test_ordering.js index 155a0c1..45c4c4b 100644 --- a/test/test_ordering.js +++ b/test/test_ordering.js @@ -9,7 +9,7 @@ describe("Compile", function() { var simpleOrderedSource = null; var complexOrderedSource = null; var inheritedSource = null; - var compileOptions = { contracts_directory: '', solc: ''}; + var compileOptions = { contracts_directory: '', solc: '', quiet: true}; describe("ABI Ordering", function(){ before("get code", function() { diff --git a/test/test_parser.js b/test/test_parser.js index 801d8a1..0b9b675 100644 --- a/test/test_parser.js +++ b/test/test_parser.js @@ -10,8 +10,8 @@ describe("Parser", function() { var solc; before("get code", async function() { - source = fs.readFileSync(path.join(__dirname, "./sources/MyContract.sol"), "utf-8"); - erroneousSource = fs.readFileSync(path.join(__dirname, "./sources/ShouldError.sol"), "utf-8"); + source = fs.readFileSync(path.join(__dirname, "./mock/MyContract.sol"), "utf-8"); + erroneousSource = fs.readFileSync(path.join(__dirname, "./mock/ShouldError.sol"), "utf-8"); const provider = new CompilerProvider(); solc = await provider.load(); diff --git a/test/test_provider.js b/test/test_provider.js index 3942b33..57949ae 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -3,9 +3,11 @@ const path = require("path"); const solc = require("solc"); const assert = require("assert"); const findCacheDir = require('find-cache-dir'); +const Resolver = require('truffle-resolver'); const compile = require("../index"); const CompilerProvider = require('../compilerProvider'); + function waitSecond() { return new Promise((resolve, reject) => setTimeout(() => resolve(), 5000)); } @@ -80,7 +82,7 @@ describe('CompilerProvider', function(){ before("get code", function() { const newPragma = fs.readFileSync(path.join(__dirname, "./sources/NewPragma.sol"), "utf-8"); - const oldPragmaPin = fs.readFileSync(path.join(__dirname, "./sources/OldPragmaPin.sol"), "utf-8"); + const oldPragmaPin = fs.readFileSync(path.join(__dirname, "./mock/OldPragmaPin.sol"), "utf-8"); const oldPragmaFloat = fs.readFileSync(path.join(__dirname, "./sources/OldPragmaFloat.sol"), "utf-8"); newPragmaSource = { "NewPragma.sol": newPragma}; @@ -246,6 +248,35 @@ describe('CompilerProvider', function(){ }); }); + it('resolves imports correctly when using built solc', function(done){ + const paths = []; + paths.push(path.join(__dirname, "./sources/ComplexOrdered.sol")); + paths.push(path.join(__dirname, "./sources/InheritB.sol")); + + let options = { + compiler : { + solc: "0.4.22", + docker: true + }, + quiet: true, + solc: '', + contracts_build_directory: path.join(__dirname, "./build"), + contracts_directory: path.join(__dirname, "./sources"), + working_directory: __dirname, + paths: paths + } + + options.resolver = new Resolver(options); + + compile.with_dependencies(options, (err, result) => { + if (err) return done(err); + + // This contract imports / inherits + assert(result['ComplexOrdered'].contract_name === 'ComplexOrdered', 'Should have compiled'); + done(); + }); + }) + it('errors if running dockerized solc without specifying an image', function(done){ options.compiler = { solc: undefined, From d7b3489169a484e511c9db9dc727f9fb7b97ad3e Mon Sep 17 00:00:00 2001 From: cgewecke Date: Tue, 1 May 2018 15:22:03 -0700 Subject: [PATCH 19/22] Add light solc wrapper for cached require speedup --- compilerProvider.js | 14 ++++---- solcWrap.js | 78 +++++++++++++++++++++++++++++++++++++++++++ test/test_provider.js | 5 ++- 3 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 solcWrap.js diff --git a/compilerProvider.js b/compilerProvider.js index 6dff396..ef627d9 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -5,6 +5,7 @@ const request = require('request-promise'); const requireFromString = require('require-from-string'); const findCacheDir = require('find-cache-dir'); const originalRequire = require('original-require'); +const solcWrap = require('./solcWrap.js'); //------------------------------ Constructor/Config ------------------------------------------------ @@ -379,9 +380,11 @@ CompilerProvider.prototype.addToCache = function(code, fileName){ * @return {Module} solc */ CompilerProvider.prototype.getFromCache = function(fileName){ - const filePath = this.resolveCache(fileName) - const cached = fs.readFileSync(filePath, "utf-8"); - return this.compilerFromString(cached); + const filePath = this.resolveCache(fileName); + const soljson = require(filePath); + const wrapped = solcWrap(soljson); + this.removeListener(); + return wrapped; } /** @@ -390,9 +393,8 @@ CompilerProvider.prototype.getFromCache = function(fileName){ * @return {Module} solc */ CompilerProvider.prototype.compilerFromString = function(code){ - const solc = this.getDefault(); - const compiler = requireFromString(code); - const wrapped = solc.setupMethods(compiler); + const soljson = requireFromString(code); + const wrapped = solcWrap(soljson); this.removeListener(); return wrapped; } diff --git a/solcWrap.js b/solcWrap.js new file mode 100644 index 0000000..68a79bd --- /dev/null +++ b/solcWrap.js @@ -0,0 +1,78 @@ +/** + * This is a truncated version of soljs's wrapper for soljson.js (the solc js file available + * at solc-bin. Using this allows us to avoid requiring solc twice, cutting ~2s off the load time) + */ + +var assert = require('assert'); + +function solcWrap (soljson) { + var compileJSON = soljson.cwrap('compileJSON', 'string', ['string', 'number']); + var compileJSONMulti = null; + if ('_compileJSONMulti' in soljson) { + compileJSONMulti = soljson.cwrap('compileJSONMulti', 'string', ['string', 'number']); + } + var compileJSONCallback = null; + var compileStandard = null; + if (('_compileJSONCallback' in soljson) || ('_compileStandard' in soljson)) { + var copyString = function (str, ptr) { + var length = soljson.lengthBytesUTF8(str); + var buffer = soljson._malloc(length + 1); + soljson.stringToUTF8(str, buffer, length + 1); + soljson.setValue(ptr, buffer, '*'); + }; + var wrapCallback = function (callback) { + assert(typeof callback === 'function', 'Invalid callback specified.'); + return function (path, contents, error) { + var result = callback(soljson.Pointer_stringify(path)); + if (typeof result.contents === 'string') { + copyString(result.contents, contents); + } + if (typeof result.error === 'string') { + copyString(result.error, error); + } + }; + }; + + // This calls compile() with args || cb + var runWithReadCallback = function (readCallback, compile, args) { + if (readCallback === undefined) { + readCallback = function (path) { + return { + error: 'File import callback not supported' + }; + }; + } + var cb = soljson.Runtime.addFunction(wrapCallback(readCallback)); + var output; + try { + args.push(cb); + output = compile.apply(undefined, args); + } catch (e) { + soljson.Runtime.removeFunction(cb); + throw e; + } + soljson.Runtime.removeFunction(cb); + return output; + }; + + var compileInternal = soljson.cwrap('compileJSONCallback', 'string', ['string', 'number', 'number']); + compileJSONCallback = function (input, optimize, readCallback) { + return runWithReadCallback(readCallback, compileInternal, [ input, optimize ]); + }; + if ('_compileStandard' in soljson) { + var compileStandardInternal = soljson.cwrap('compileStandard', 'string', ['string', 'number']); + compileStandard = function (input, readCallback) { + return runWithReadCallback(readCallback, compileStandardInternal, [ input ]); + }; + } + } + + var version = soljson.cwrap('version', 'string', []); + + return { + compileStandard: compileStandard, + version: version, + }; +} + +module.exports = solcWrap; \ No newline at end of file diff --git a/test/test_provider.js b/test/test_provider.js index 57949ae..dc0b7b2 100644 --- a/test/test_provider.js +++ b/test/test_provider.js @@ -9,7 +9,7 @@ const CompilerProvider = require('../compilerProvider'); function waitSecond() { - return new Promise((resolve, reject) => setTimeout(() => resolve(), 5000)); + return new Promise((resolve, reject) => setTimeout(() => resolve(), 1250)); } describe('CompilerProvider', function(){ @@ -179,6 +179,9 @@ describe('CompilerProvider', function(){ const thunk = findCacheDir({name: 'truffle', thunk: true}); const expectedCache = thunk('soljson-v0.4.21+commit.dfe3193c.js'); + // Delete if it's already there. + if (fs.existsSync(expectedCache)) fs.unlinkSync(expectedCache); + options.compiler = { cache: true, solc: "0.4.21" From d4a74649b655cca86007a3590522f66ed24a8bb0 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Tue, 1 May 2018 16:05:56 -0700 Subject: [PATCH 20/22] Cache docker version strings --- compilerProvider.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index ef627d9..d2fbedb 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -278,6 +278,13 @@ CompilerProvider.prototype.isLocal = function(localPath){ */ CompilerProvider.prototype.validateDocker = function(){ const image = this.config.solc; + const fileName = image + '.version'; + + // Skip validation if they've validated for this image before. + if (this.isCached(fileName)){ + const cachePath = this.resolveCache(fileName); + return fs.readFileSync(cachePath, 'utf-8'); + } // Image specified if (!image) throw this.errors('noString', image); @@ -296,9 +303,11 @@ CompilerProvider.prototype.validateDocker = function(){ throw this.errors('noImage', image); } - // Get version + // Get version & cache. const version = child.execSync('docker run ethereum/solc:' + image + ' --version'); - return this.normalizeVersion(version); + const normalized = this.normalizeVersion(version); + this.addToCache(normalized, fileName); + return normalized; } /** @@ -341,6 +350,7 @@ CompilerProvider.prototype.normalizeVersion = function(version){ return version.split(':')[1].trim(); } + /** * Returns path to cached solc version * @param {String} fileName ex: "soljson-v0.4.21+commit.dfe3193c.js" From 0cbc35254b1c89a3542921ac3fd442d13e77854c Mon Sep 17 00:00:00 2001 From: cgewecke Date: Tue, 1 May 2018 19:23:32 -0700 Subject: [PATCH 21/22] Use originalRequire to prevent webpack mangle --- compilerProvider.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compilerProvider.js b/compilerProvider.js index d2fbedb..c7300f2 100644 --- a/compilerProvider.js +++ b/compilerProvider.js @@ -125,7 +125,7 @@ CompilerProvider.prototype.getDockerTags = function(){ //------------------------------------ Getters ----------------------------------------------------- /** - * Gets solc from local `node_modules`. Equivalent to `require("solc")` + * Gets solc from `node_modules`.` * @return {Module} solc */ CompilerProvider.prototype.getDefault = function(){ @@ -135,7 +135,7 @@ CompilerProvider.prototype.getDefault = function(){ } /** - * Gets an npm installed solc from specified absolute path. + * Gets an npm installed solc from specified path. * @param {String} localPath * @return {Module} */ @@ -391,7 +391,7 @@ CompilerProvider.prototype.addToCache = function(code, fileName){ */ CompilerProvider.prototype.getFromCache = function(fileName){ const filePath = this.resolveCache(fileName); - const soljson = require(filePath); + const soljson = originalRequire(filePath); const wrapped = solcWrap(soljson); this.removeListener(); return wrapped; From c7a997a962093de394d116b8658352a473446bf5 Mon Sep 17 00:00:00 2001 From: cgewecke Date: Wed, 2 May 2018 10:51:34 -0700 Subject: [PATCH 22/22] Cleanup solcWrap --- solcWrap.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/solcWrap.js b/solcWrap.js index 68a79bd..0d6504a 100644 --- a/solcWrap.js +++ b/solcWrap.js @@ -55,12 +55,24 @@ function solcWrap (soljson) { return output; }; - var compileInternal = soljson.cwrap('compileJSONCallback', 'string', ['string', 'number', 'number']); + var compileInternal = soljson.cwrap( + 'compileJSONCallback', + 'string', + ['string', 'number', 'number'] + ); + compileJSONCallback = function (input, optimize, readCallback) { return runWithReadCallback(readCallback, compileInternal, [ input, optimize ]); }; + if ('_compileStandard' in soljson) { - var compileStandardInternal = soljson.cwrap('compileStandard', 'string', ['string', 'number']); + + var compileStandardInternal = soljson.cwrap( + 'compileStandard', + 'string', + ['string', 'number'] + ); + compileStandard = function (input, readCallback) { return runWithReadCallback(readCallback, compileStandardInternal, [ input ]); }; @@ -75,4 +87,4 @@ function solcWrap (soljson) { }; } -module.exports = solcWrap; \ No newline at end of file +module.exports = solcWrap;