diff --git a/source/dub/commandline.d b/source/dub/commandline.d index 628c1e0..cce2440 100644 --- a/source/dub/commandline.d +++ b/source/dub/commandline.d @@ -94,7 +94,8 @@ new BuildCommand, new TestCommand, new GenerateCommand, - new DescribeCommand + new DescribeCommand, + new DustmiteCommand ), CommandGroup("Package management", new FetchCommand, @@ -157,15 +158,19 @@ return 1; } - // initialize DUB - auto package_suppliers = registry_urls.map!(url => cast(PackageSupplier)new RegistryPackageSupplier(Url(url))).array; - Dub dub = new Dub(package_suppliers, root_path); - dub.dryRun = annotate; + Dub dub; + + if (!cmd.skipDubInitialization) { + // initialize DUB + auto package_suppliers = registry_urls.map!(url => cast(PackageSupplier)new RegistryPackageSupplier(Url(url))).array; + dub = new Dub(package_suppliers, root_path); + dub.dryRun = annotate; - // make the CWD package available so that for example sub packages can reference their - // parent package. - try dub.packageManager.getTemporaryPackage(Path(root_path)); - catch (Exception e) { logDiagnostic("No package found in current working directory."); } + // make the CWD package available so that for example sub packages can reference their + // parent package. + try dub.packageManager.getTemporaryPackage(Path(root_path)); + catch (Exception e) { logDiagnostic("No package found in current working directory."); } + } try return cmd.execute(dub, remaining_args, app_args); catch (UsageException e) { @@ -243,6 +248,7 @@ string[] helpText; bool acceptsAppArgs; bool hidden = false; // used for deprecated commands + bool skipDubInitialization = false; abstract void prepare(scope CommandArgs args); abstract int execute(Dub dub, string[] free_args, string[] app_args); @@ -998,7 +1004,7 @@ foreach (p; dub.packageManager.getPackageIterator()) logInfo(" %s %s: %s", p.name, p.ver, p.path.toNativeString()); logInfo(""); - return true; + return 0; } } @@ -1014,6 +1020,178 @@ /******************************************************************************/ +/* DUSTMITE */ +/******************************************************************************/ + +class DustmiteCommand : PackageBuildCommand { + private { + int m_compilerStatusCode = int.min; + int m_linkerStatusCode = int.min; + int m_programStatusCode = int.min; + string m_compilerRegex; + string m_linkerRegex; + string m_programRegex; + string m_testPackage; + bool m_combined; + } + + this() + { + this.name = "dustmite"; + this.argumentsPattern = ""; + this.acceptsAppArgs = true; + this.description = "Create reduced test cases for build errors"; + this.helpText = [ + "This command uses the Dustmite utility to isolate the cause of build errors in a DUB project.", + "", + "It will create a copy of all involved packages and run dustmite on this copy, leaving a reduced test case.", + "", + "Determining the desired error condition is done by checking the compiler/linker status code, as well as their output (stdout and stderr combined). If --program-status or --program-regex is given and the generated binary is an executable, it will be executed and its output will also be incorporated into the final decision." + ]; + } + + override void prepare(scope CommandArgs args) + { + args.getopt("compiler-status", &m_compilerStatusCode, ["The expected status code of the compiler run"]); + args.getopt("compiler-regex", &m_compilerRegex, ["A regular expression used to match against the compiler output"]); + args.getopt("linker-status", &m_linkerStatusCode, ["The expected status code of the liner run"]); + args.getopt("linker-regex", &m_linkerRegex, ["A regular expression used to match against the linker output"]); + args.getopt("program-status", &m_programStatusCode, ["The expected status code of the built executable"]); + args.getopt("program-regex", &m_programRegex, ["A regular expression used to match against the program output"]); + args.getopt("test-package", &m_testPackage, ["Perform a test run - usually only used internally"]); + args.getopt("combined", &m_combined, ["Builds multiple packages with one compiler run"]); + super.prepare(args); + + // speed up loading when in test mode + if (m_testPackage.length) skipDubInitialization = true; + } + + override int execute(Dub dub, string[] free_args, string[] app_args) + { + if (m_testPackage.length) { + dub = new Dub(Path(getcwd())); + + setupPackage(dub, m_testPackage); + m_defaultConfig = dub.project.getDefaultConfiguration(m_buildPlatform); + + GeneratorSettings gensettings; + gensettings.platform = m_buildPlatform; + gensettings.config = m_build_config.length ? m_build_config : m_defaultConfig; + gensettings.buildType = m_build_type; + gensettings.compiler = m_compiler; + gensettings.buildSettings = m_buildSettings; + gensettings.combined = m_combined; + gensettings.run = m_programStatusCode != int.min || m_programRegex.length; + gensettings.runArgs = app_args; + gensettings.force = true; + gensettings.compileCallback = check(m_compilerStatusCode, m_compilerRegex); + gensettings.linkCallback = check(m_linkerStatusCode, m_linkerRegex); + gensettings.runCallback = check(m_programStatusCode, m_programRegex); + try dub.generateProject("build", gensettings); + catch (DustmiteMismatchException) { + logInfo("Dustmite test doesn't match."); + return 3; + } + catch (DustmiteMatchException) { + logInfo("Dustmite test matches."); + return 0; + } + } else { + enforceUsage(free_args.length == 1, "Expected destination path."); + auto path = Path(free_args[0]); + path.normalize(); + enforceUsage(path.length > 0, "Destination path must not be empty."); + if (!path.absolute) path = Path(getcwd()) ~ path; + enforceUsage(!path.startsWith(dub.rootPath), "Destination path must not be a sub directory of the tested package!"); + + setupPackage(dub, null); + auto prj = dub.project; + if (m_build_config.empty) + m_build_config = prj.getDefaultConfiguration(m_buildPlatform); + + void copyFolderRec(Path folder, Path dstfolder) + { + mkdirRecurse(dstfolder.toNativeString()); + foreach (de; iterateDirectory(folder.toNativeString())) { + if (de.name.startsWith(".")) continue; + if (de.isDirectory) { + copyFolderRec(folder ~ de.name, dstfolder ~ de.name); + } else { + if (de.name.endsWith(".o") || de.name.endsWith(".obj")) continue; + if (de.name.endsWith(".exe")) continue; + try copyFile(folder ~ de.name, dstfolder ~ de.name); + catch (Exception e) { + logWarn("Failed to copy file %s: %s", (folder ~ de.name).toNativeString(), e.msg); + } + } + } + } + + bool[string] visited; + foreach (pack_; prj.getTopologicalPackageList()) { + auto pack = pack_.basePackage; + if (pack.name in visited) continue; + visited[pack.name] = true; + logInfo("Copy package '%s' to destination folder...", pack.name); + copyFolderRec(pack.path, path ~ pack.name); + } + logInfo("Executing dustmite..."); + auto testcmd = format("dub dustmite --vquiet --test-package=%s", prj.name); + if (m_compilerStatusCode != int.min) testcmd ~= format(" --compiler-status=%s", m_compilerStatusCode); + if (m_compilerRegex.length) testcmd ~= format(" \"--compiler-regex=%s\"", m_compilerRegex); + if (m_linkerStatusCode != int.min) testcmd ~= format(" --linker-status=%s", m_linkerStatusCode); + if (m_linkerRegex.length) testcmd ~= format(" \"--linker-regex=%s\"", m_linkerRegex); + if (m_programStatusCode != int.min) testcmd ~= format(" --program-status=%s", m_programStatusCode); + if (m_programRegex.length) testcmd ~= format(" \"--program-regex=%s\"", m_programRegex); + if (m_combined) testcmd ~= " --combined"; + // TODO: pass all original parameters + auto dmpid = spawnProcess(["dustmite", path.toNativeString(), testcmd]); + return dmpid.wait(); + } + return 0; + } + + void delegate(int, string) check(int code_match, string regex_match) + { + return (code, output) { + import std.encoding; + import std.regex; + + if (code_match != int.min && code != code_match) { + logInfo("Exit code %s doesn't match expected value %s", code, code_match); + throw new DustmiteMismatchException; + } + + if (regex_match.length > 0 && !match(output.sanitize, regex_match)) { + logInfo("Output doesn't match regex:"); + logInfo("%s", output); + throw new DustmiteMismatchException; + } + + if (code != 0 && code_match != int.min || regex_match.length > 0) { + logInfo("Tool failed, but matched either exit code or output - counting as match."); + throw new DustmiteMatchException; + } + }; + } + + static class DustmiteMismatchException : Exception { + this(string message = "", string file = __FILE__, int line = __LINE__, Throwable next = null) + { + super(message, file, line, next); + } + } + + static class DustmiteMatchException : Exception { + this(string message = "", string file = __FILE__, int line = __LINE__, Throwable next = null) + { + super(message, file, line, next); + } + } +} + + +/******************************************************************************/ /* HELP */ /******************************************************************************/ diff --git a/source/dub/compilers/compiler.d b/source/dub/compilers/compiler.d index 89692fa..014e66d 100644 --- a/source/dub/compilers/compiler.d +++ b/source/dub/compilers/compiler.d @@ -202,10 +202,24 @@ void setTarget(ref BuildSettings settings, in BuildPlatform platform); /// Invokes the compiler using the given flags - void invoke(in BuildSettings settings, in BuildPlatform platform); + void invoke(in BuildSettings settings, in BuildPlatform platform, void delegate(int, string) output_callback); /// Invokes the underlying linker directly - void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects); + void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects, void delegate(int, string) output_callback); + + protected final void invokeTool(string[] args, void delegate(int, string) output_callback) + { + int status; + if (output_callback) { + auto result = execute(args); + output_callback(result.status, result.output); + status = result.status; + } else { + auto compiler_pid = spawnProcess(args); + status = compiler_pid.wait(); + } + enforce(status == 0, args[0] ~ " failed with exit code "~to!string(status)); + } } /// BuildPlatform specific settings, like needed libraries or additional diff --git a/source/dub/compilers/dmd.d b/source/dub/compilers/dmd.d index bd2fce4..f5df9f3 100644 --- a/source/dub/compilers/dmd.d +++ b/source/dub/compilers/dmd.d @@ -161,19 +161,17 @@ settings.addDFlags("-of"~tpath.toNativeString()); } - void invoke(in BuildSettings settings, in BuildPlatform platform) + void invoke(in BuildSettings settings, in BuildPlatform platform, void delegate(int, string) output_callback) { auto res_file = getTempDir() ~ ("dub-build-"~uniform(0, uint.max).to!string~"-.rsp"); std.file.write(res_file.toNativeString(), join(settings.dflags.map!(s => s.canFind(' ') ? "\""~s~"\"" : s), "\n")); scope (exit) remove(res_file.toNativeString()); logDiagnostic("%s %s", platform.compilerBinary, join(cast(string[])settings.dflags, " ")); - auto compiler_pid = spawnProcess([platform.compilerBinary, "@"~res_file.toNativeString()]); - auto result = compiler_pid.wait(); - enforce(result == 0, "DMD compile run failed with exit code "~to!string(result)); + invokeTool([platform.compilerBinary, "@"~res_file.toNativeString()], output_callback); } - void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects) + void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects, void delegate(int, string) output_callback) { import std.string; auto tpath = Path(settings.targetPath) ~ getTargetFileName(settings, platform); @@ -184,8 +182,7 @@ args ~= settings.lflags.map!(l => "-L"~l)().array; args ~= settings.dflags.filter!(f => isLinkerDFlag(f)).array; logDiagnostic("%s", args.join(" ")); - auto res = spawnProcess(args).wait(); - enforce(res == 0, "Link command failed with exit code "~to!string(res)); + invokeTool(args, output_callback); } private static bool isLinkerDFlag(string arg) diff --git a/source/dub/compilers/gdc.d b/source/dub/compilers/gdc.d index 7a048cf..642aa0d 100644 --- a/source/dub/compilers/gdc.d +++ b/source/dub/compilers/gdc.d @@ -149,19 +149,17 @@ settings.addDFlags("-o", tpath.toNativeString()); } - void invoke(in BuildSettings settings, in BuildPlatform platform) + void invoke(in BuildSettings settings, in BuildPlatform platform, void delegate(int, string) output_callback) { auto res_file = getTempDir() ~ ("dub-build-"~uniform(0, uint.max).to!string~"-.rsp"); std.file.write(res_file.toNativeString(), join(settings.dflags.map!(s => escape(s)), "\n")); scope (exit) remove(res_file.toNativeString()); logDiagnostic("%s %s", platform.compilerBinary, join(cast(string[])settings.dflags, " ")); - auto compiler_pid = spawnProcess([platform.compilerBinary, "@"~res_file.toNativeString()]); - auto result = compiler_pid.wait(); - enforce(result == 0, "GDC compile run failed with exit code "~to!string(result)); + invokeTool([platform.compilerBinary, "@"~res_file.toNativeString()], output_callback); } - void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects) + void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects, void delegate(int, string) output_callback) { assert(false, "Separate linking not implemented for GDC"); } diff --git a/source/dub/compilers/ldc.d b/source/dub/compilers/ldc.d index d670f41..8539250 100644 --- a/source/dub/compilers/ldc.d +++ b/source/dub/compilers/ldc.d @@ -161,19 +161,17 @@ settings.addDFlags("-of"~tpath.toNativeString()); } - void invoke(in BuildSettings settings, in BuildPlatform platform) + void invoke(in BuildSettings settings, in BuildPlatform platform, void delegate(int, string) output_callback) { auto res_file = getTempDir() ~ ("dub-build-"~uniform(0, uint.max).to!string~"-.rsp"); std.file.write(res_file.toNativeString(), join(cast(string[])settings.dflags, "\n")); scope (exit) remove(res_file.toNativeString()); logDiagnostic("%s %s", platform.compilerBinary, join(cast(string[])settings.dflags, " ")); - auto compiler_pid = spawnProcess([platform.compilerBinary, "@"~res_file.toNativeString()]); - auto result = compiler_pid.wait(); - enforce(result == 0, "LDC compile run failed with exit code "~to!string(result)); + invokeTool([platform.compilerBinary, "@"~res_file.toNativeString()], output_callback); } - void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects) + void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects, void delegate(int, string) output_callback) { assert(false, "Separate linking not implemented for GDC"); } diff --git a/source/dub/dub.d b/source/dub/dub.d index f0c58ec..07f8506 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -57,6 +57,7 @@ Json m_systemConfig, m_userConfig; Path m_projectPath; Project m_project; + Path m_overrideSearchPath; } /// Initiales the package manager for the vibe application @@ -93,7 +94,15 @@ ps ~= defaultPackageSuppliers(); m_packageSuppliers = ps; - m_packageManager = new PackageManager(m_userDubPath, m_systemDubPath); + m_packageManager = new PackageManager(m_userDubPath, m_systemDubPath, false); + updatePackageSearchPath(); + } + + /// Initializes DUB with only a single search path + this(Path override_path) + { + m_overrideSearchPath = override_path; + m_packageManager = new PackageManager(Path(), Path(), false); updatePackageSearchPath(); } @@ -144,6 +153,13 @@ m_project = new Project(m_packageManager, pack); } + void overrideSearchPath(Path path) + { + if (!path.absolute) path = Path(getcwd()) ~ path; + m_overrideSearchPath = path; + updatePackageSearchPath(); + } + string getDefaultConfiguration(BuildPlatform platform, bool allow_non_library_configs = true) const { return m_project.getDefaultConfiguration(platform, allow_non_library_configs); } /// Performs retrieval and removal as necessary for @@ -552,13 +568,19 @@ private void updatePackageSearchPath() { - auto p = environment.get("DUBPATH"); - Path[] paths; + if (m_overrideSearchPath.length) { + m_packageManager.disableDefaultSearchPaths = true; + m_packageManager.searchPath = [m_overrideSearchPath]; + } else { + auto p = environment.get("DUBPATH"); + Path[] paths; - version(Windows) enum pathsep = ";"; - else enum pathsep = ":"; - if (p.length) paths ~= p.split(pathsep).map!(p => Path(p))().array(); - m_packageManager.searchPath = paths; + version(Windows) enum pathsep = ";"; + else enum pathsep = ":"; + if (p.length) paths ~= p.split(pathsep).map!(p => Path(p))().array(); + m_packageManager.disableDefaultSearchPaths = false; + m_packageManager.searchPath = paths; + } } private Path makeAbsolute(Path p) const { return p.absolute ? p : m_rootPath ~ p; } diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 87612ff..b06f6ef 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -75,7 +75,7 @@ auto buildsettings = targets[m_project.mainPackage.name].buildSettings; if (settings.run && !(buildsettings.options & BuildOptions.syntaxOnly)) { auto exe_file_path = Path(buildsettings.targetPath) ~ getTargetFileName(buildsettings, settings.platform); - runTarget(exe_file_path, buildsettings, settings.runArgs); + runTarget(exe_file_path, buildsettings, settings.runArgs, settings); } } @@ -359,7 +359,7 @@ // invoke the compiler logInfo("Running %s...", settings.platform.compilerBinary); - settings.compiler.invoke(buildsettings, settings.platform); + settings.compiler.invoke(buildsettings, settings.platform, settings.compileCallback); } else { // determine path for the temporary object file string tempobjname = buildsettings.targetName; @@ -381,14 +381,14 @@ settings.compiler.prepareBuildSettings(buildsettings, BuildSetting.commandLine); logInfo("Compiling..."); - settings.compiler.invoke(buildsettings, settings.platform); + settings.compiler.invoke(buildsettings, settings.platform, settings.compileCallback); logInfo("Linking..."); - settings.compiler.invokeLinker(lbuildsettings, settings.platform, [tempobj.toNativeString()]); + settings.compiler.invokeLinker(lbuildsettings, settings.platform, [tempobj.toNativeString()], settings.linkCallback); } } - void runTarget(Path exe_file_path, in BuildSettings buildsettings, string[] run_args) + void runTarget(Path exe_file_path, in BuildSettings buildsettings, string[] run_args, GeneratorSettings settings) { if (buildsettings.targetType == TargetType.executable) { auto cwd = Path(getcwd()); @@ -411,9 +411,14 @@ exe_path_string = ".\\" ~ exe_path_string; } logInfo("Running %s %s", exe_path_string, run_args.join(" ")); - auto prg_pid = spawnProcess(exe_path_string ~ run_args); - auto result = prg_pid.wait(); - enforce(result == 0, "Program exited with code "~to!string(result)); + if (settings.runCallback) { + auto res = execute(exe_path_string ~ run_args); + settings.runCallback(res.status, res.output); + } else { + auto prg_pid = spawnProcess(exe_path_string ~ run_args); + auto result = prg_pid.wait(); + enforce(result == 0, "Program exited with code "~to!string(result)); + } } else logInfo("Target is a library. Skipping execution."); } diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index 349a51e..d69bd34 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -192,6 +192,9 @@ // only used for generator "build" bool run, force, direct, clean, rdmd; string[] runArgs; + void delegate(int status, string output) compileCallback; + void delegate(int status, string output) linkCallback; + void delegate(int status, string output) runCallback; } diff --git a/source/dub/packagemanager.d b/source/dub/packagemanager.d index 5584638..447a079 100644 --- a/source/dub/packagemanager.d +++ b/source/dub/packagemanager.d @@ -56,26 +56,31 @@ Path[] m_searchPath; Package[] m_packages; Package[] m_temporaryPackages; + bool m_disableDefaultSearchPaths = false; } - this(Path user_path, Path system_path) + this(Path user_path, Path system_path, bool refresh_packages = true) { m_repositories[LocalPackageType.user] = Repository(user_path); m_repositories[LocalPackageType.system] = Repository(system_path); - refresh(true); + if (refresh_packages) refresh(true); } @property void searchPath(Path[] paths) { m_searchPath = paths.dup; refresh(false); } @property const(Path)[] searchPath() const { return m_searchPath; } + @property void disableDefaultSearchPaths(bool val) { m_disableDefaultSearchPaths = val; refresh(true); } + @property const(Path)[] completeSearchPath() const { auto ret = appender!(Path[])(); ret.put(m_searchPath); - ret.put(m_repositories[LocalPackageType.user].searchPath); - ret.put(m_repositories[LocalPackageType.user].packagePath); - ret.put(m_repositories[LocalPackageType.system].searchPath); - ret.put(m_repositories[LocalPackageType.system].packagePath); + if (!m_disableDefaultSearchPaths) { + ret.put(m_repositories[LocalPackageType.user].searchPath); + ret.put(m_repositories[LocalPackageType.user].packagePath); + ret.put(m_repositories[LocalPackageType.system].searchPath); + ret.put(m_repositories[LocalPackageType.system].packagePath); + } return ret.data; } @@ -470,7 +475,7 @@ Path list_path = m_repositories[type].packagePath; Package[] packs; Path[] paths; - try { + if (!m_disableDefaultSearchPaths) try { auto local_package_file = list_path ~ LocalPackagesFilename; logDiagnostic("Looking for local package map at %s", local_package_file.toNativeString()); if( !existsFile(local_package_file) ) return;