diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c652b2..9ab5b69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,10 @@ Every feature addition should come with a changelog entry, see the [changelog/README.md](changelog/README.md) for how to add a new entry. Any `.dd` file is rendered using ddoc and is the same format used across all dlang repos. For bugfixes make sure to automatically [close the issue via commit message](https://blog.github.com/2013-01-22-closing-issues-via-commit-messages/) (e.g. `fixes #123`), so that they can be listed in the changelog. +## Documentation + +The source of the documentation for DUB lives in a separate repository: https://github.com/dlang/dub-docs. + ## Backwards compatiblity DUB is a command line tool, as well as a library that can be embedded into other applications. We aim to stay backwards compatible as long as possible and as required by the SemVer specification. For this reason, any change to the public API, as well as to the command line interface, needs to be carefully reviewed for possible breaking changes. No breaking changes are allowed to enter the master branch at this point. diff --git a/README.md b/README.md index fbad53d..f1badb6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ There is a central [package registry](https://github.com/dlang/dub-registry/) located at . -[![GitHub tag](https://img.shields.io/github/tag/dlang/dub.svg?maxAge=86400)](#) [![Travis](https://travis-ci.com/dlang/dub.svg?branch=master)](https://travis-ci.com/dlang/dub) [![Coverage Status](https://coveralls.io/repos/dlang/dub/badge.svg)](https://coveralls.io/r/dlang/dub) +[![GitHub tag](https://img.shields.io/github/tag/dlang/dub.svg?maxAge=86400)](#) [![Coverage Status](https://coveralls.io/repos/dlang/dub/badge.svg)](https://coveralls.io/r/dlang/dub) [![Buildkite](https://badge.buildkite.com/c54d71c42284a042b9d578e28e093dff35f20cc8528319b1b6.svg?branch=master)](https://buildkite.com/dlang/dub) ## Introduction diff --git a/changelog/actual-dynamiclibrary.dd b/changelog/actual-dynamiclibrary.dd new file mode 100644 index 0000000..15757e0 --- /dev/null +++ b/changelog/actual-dynamiclibrary.dd @@ -0,0 +1,7 @@ +Builds dynamicLibrary targets as dynamic libraries instead of static libraries. + +Dub will no longer build dynamicLibrary targetType's as staticLibrary. + +Except for x86_omf. This has been disabled due to numerous issues that will lead this to not doing what is expected of it. + +No compiler or linker flags have been added at this time, you will need to specify the relevant flag to get the compiler to link dynamically against Phobos. diff --git a/changelog/build-path-variable.dd b/changelog/build-path-variable.dd new file mode 100644 index 0000000..f2f59e4 --- /dev/null +++ b/changelog/build-path-variable.dd @@ -0,0 +1,15 @@ +The `$DUB_BUILD_PATH` variable was added + +The `$DUB_BUILD_PATH` variable is now defined inside the `postBuildCommands` +section. It contains the absolute path in which the package was built, and can +be used to copy by-products of the build process to their intended locations. + +For example, if an executable exports symbols, you will want to make the +resulting import library and symbols export file available somewhere. That can +be done with a `dub.json` section like this: +------- + "postBuildCommands-windows": [ + "copy /y $DUB_BUILD_PATH\\$DUB_TARGET_NAME.lib $PACKAGE_DIR\\lib" + "copy /y $DUB_BUILD_PATH\\$DUB_TARGET_NAME.exp $PACKAGE_DIR\\lib" + ], +------- diff --git a/changelog/command-variables.dd b/changelog/command-variables.dd new file mode 100644 index 0000000..9d30128 --- /dev/null +++ b/changelog/command-variables.dd @@ -0,0 +1,20 @@ +Command environment variable substitution changed + +Now users can use the documented predefined variables inside custom command +directives without the need for a wrapper shell script. + +Before this would have failed: +```json +"preBuildCommands": ["$DC -run foo.d"] +``` +unless DC was defined as environment variable outside DUB. + +It was before possible to run a script that used the $DC environment variable or +on POSIX escape the `$` with `$$DC` to make the shell substitute the variable. +These workarounds are no longer needed now. + +API change: none of the different command directives are no longer substituted +with the process environment variables. You now access the raw commands as +provided by the user in the recipe. `dub describe` has been adjusted and now +also processes the predefined environment variables as well as the process +environment variables. diff --git a/changelog/etc-dub-settings-json.dd b/changelog/etc-dub-settings-json.dd new file mode 100644 index 0000000..0699389 --- /dev/null +++ b/changelog/etc-dub-settings-json.dd @@ -0,0 +1,9 @@ +Posix: use /etc/dub/settings.json if DUB is installed in /usr + +For Linux distributions that put the dub installation in /usr, there is now a +special case that DUB will load from /etc/dub/settings.json (absolute path) if +the installation is inside /usr. + +Previously settings would have attempted to be loaded from +`/usr/etc/dub/settings.json` if installed in `/usr/bin/dub`. This is still +loaded if it exists, but if not `/etc/dub/settings.json` will be loaded. diff --git a/changelog/injectsourcefile.dd b/changelog/injectsourcefile.dd new file mode 100644 index 0000000..5421ddc --- /dev/null +++ b/changelog/injectsourcefile.dd @@ -0,0 +1,9 @@ +Adds injection of source files from dependencies via injectSourceFiles command + +Each (sub)package now supports a source file that will be included in any executable or dynamic library that depends either directly or indirectly on it. + +This can be used to register and unregister elements of a package within the dependant package without requiring the dependant to acknowledge that the registration mechanism needs to take place. + +A comparable existing feature to this is the usage of sourceLibrary target type. +A sourceLibrary targetType defers compilation of source code until it is dependent upon by a static library, dynamic library or executable (sub)package. +Unlike sourceLibrary the injection of source code using this feature will inject it into every dynamic library and executable that depends on it, regardless of how deep it is in the dependency graph. diff --git a/examples/injected-from-dependency/dependency/ahook.d b/examples/injected-from-dependency/dependency/ahook.d new file mode 100644 index 0000000..d894409 --- /dev/null +++ b/examples/injected-from-dependency/dependency/ahook.d @@ -0,0 +1,14 @@ +module ahook; + +version(D_BetterC) { + pragma(crt_constructor) + extern(C) void someInitializer() { + import core.stdc.stdio; + printf("Hook ran!\n"); + } +} else { + shared static this() { + import std.stdio; + writeln("We have a runtime!!!!"); + } +} diff --git a/examples/injected-from-dependency/dependency/dub.json b/examples/injected-from-dependency/dependency/dub.json new file mode 100644 index 0000000..419a4af --- /dev/null +++ b/examples/injected-from-dependency/dependency/dub.json @@ -0,0 +1,9 @@ +{ + "name": "toload", + "description": "Example to showcase injection of a source file from a dependency dependency.", + "targetType": "library", + "buildOptions": ["betterC"], + "sourcePaths": ["source"], + "importPaths": ["source"], + "injectSourceFiles": ["ahook.d"] +} diff --git a/examples/injected-from-dependency/dependency/source/something.d b/examples/injected-from-dependency/dependency/source/something.d new file mode 100644 index 0000000..c21c87d --- /dev/null +++ b/examples/injected-from-dependency/dependency/source/something.d @@ -0,0 +1,11 @@ +module something; + +void doSomething() { + import core.stdc.stdio; + + version(D_BetterC) { + printf("druntime is not in the executable :(\n"); + } else { + printf("druntime is in executable!\n"); + } +} diff --git a/examples/injected-from-dependency/usage/dub.json b/examples/injected-from-dependency/usage/dub.json new file mode 100644 index 0000000..7e2b410 --- /dev/null +++ b/examples/injected-from-dependency/usage/dub.json @@ -0,0 +1,8 @@ +{ + "name": "runner", + "description": "Example to showcase injection of a source file from a dependency runner.", + "targetType": "executable", + "dependencies": { + "toload": {"path": "../dependency"} + } +} \ No newline at end of file diff --git a/examples/injected-from-dependency/usage/source/entry.d b/examples/injected-from-dependency/usage/source/entry.d new file mode 100644 index 0000000..6b4ed7c --- /dev/null +++ b/examples/injected-from-dependency/usage/source/entry.d @@ -0,0 +1,4 @@ +void main() { + import something; + doSomething(); +} diff --git a/source/dub/commandline.d b/source/dub/commandline.d index 6d0bc45..d7be6fe 100644 --- a/source/dub/commandline.d +++ b/source/dub/commandline.d @@ -1536,7 +1536,7 @@ "target-type, target-path, target-name, working-directory, " ~ "copy-files, string-import-files, pre-generate-commands, " ~ "post-generate-commands, pre-build-commands, post-build-commands, " ~ - "requirements", + "pre-run-commands, post-run-commands, requirements", ]; } diff --git a/source/dub/compilers/buildsettings.d b/source/dub/compilers/buildsettings.d index 594d38f..212c22a 100644 --- a/source/dub/compilers/buildsettings.d +++ b/source/dub/compilers/buildsettings.d @@ -31,6 +31,7 @@ string[] libs; string[] linkerFiles; string[] sourceFiles; + string[] injectSourceFiles; string[] copyFiles; string[] extraDependencyFiles; string[] versions; @@ -115,6 +116,7 @@ void addSourceFiles(in string[] value...) { add(sourceFiles, value); } void prependSourceFiles(in string[] value...) { prepend(sourceFiles, value); } void removeSourceFiles(in string[] value...) { removePaths(sourceFiles, value); } + void addInjectSourceFiles(in string[] value...) { add(injectSourceFiles, value); } void addCopyFiles(in string[] value...) { add(copyFiles, value); } void addExtraDependencyFiles(in string[] value...) { add(extraDependencyFiles, value); } void addVersions(in string[] value...) { add(versions, value); } diff --git a/source/dub/compilers/compiler.d b/source/dub/compilers/compiler.d index bb62e16..950f29a 100644 --- a/source/dub/compilers/compiler.d +++ b/source/dub/compilers/compiler.d @@ -38,6 +38,16 @@ } } +/// Exception thrown in invokeTool and probePlatform if running the compiler +/// returned non-zero exit code. +class CompilerInvocationException : Exception +{ + this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable nextInChain = null) pure nothrow @safe + { + super(msg, file, line, nextInChain); + } +} + /** Returns a compiler handler for a given binary name. The name will be compared against the canonical name of each registered @@ -128,10 +138,12 @@ } version (Posix) if (status == -9) { - throw new Exception(format("%s failed with exit code %s. This may indicate that the process has run out of memory.", - args[0], status)); + throw new CompilerInvocationException( + format("%s failed with exit code %s. This may indicate that the process has run out of memory.", + args[0], status)); } - enforce(status == 0, format("%s failed with exit code %s.", args[0], status)); + enforce!CompilerInvocationException(status == 0, + format("%s failed with exit code %s.", args[0], status)); } /** Compiles platform probe file with the specified compiler and parses its output. @@ -149,7 +161,8 @@ auto fil = generatePlatformProbeFile(); auto result = executeShell(escapeShellCommand(compiler_binary ~ args ~ fil.toNativeString())); - enforce(result.status == 0, format("Failed to invoke the compiler %s to determine the build platform: %s", + enforce!CompilerInvocationException(result.status == 0, + format("Failed to invoke the compiler %s to determine the build platform: %s", compiler_binary, result.output)); auto build_platform = readPlatformJsonProbe(result.output); @@ -169,7 +182,9 @@ // cmdline option does not lead to the same string being found among // `build_platform.architecture`, as it's brittle and doesn't work with triples. if (build_platform.compiler != "ldc") { - if (arch_override.length && !build_platform.architecture.canFind(arch_override)) { + if (arch_override.length && !build_platform.architecture.canFind(arch_override) && + !(build_platform.compiler == "dmd" && arch_override.among("x86_omf", "x86_mscoff")) // Will be fixed in determinePlatform + ) { logWarn(`Failed to apply the selected architecture %s. Got %s.`, arch_override, build_platform.architecture); } diff --git a/source/dub/compilers/dmd.d b/source/dub/compilers/dmd.d index d92d045..65ee2f4 100644 --- a/source/dub/compilers/dmd.d +++ b/source/dub/compilers/dmd.d @@ -106,29 +106,70 @@ BuildPlatform determinePlatform(ref BuildSettings settings, string compiler_binary, string arch_override) { + // Set basic arch flags for the probe - might be revised based on the exact value + compiler version string[] arch_flags; - switch (arch_override) { - default: throw new UnsupportedArchitectureException(arch_override); - case "": - // Don't use Optlink by default on Windows - version (Windows) { - const is64bit = isWow64(); - if (!is64bit.isNull) - arch_flags = [is64bit.get ? "-m64" : "-m32mscoff"]; - } - break; - case "x86": arch_flags = ["-m32"]; break; - case "x86_omf": arch_flags = ["-m32"]; break; - case "x86_64": arch_flags = ["-m64"]; break; - case "x86_mscoff": arch_flags = ["-m32mscoff"]; break; + if (arch_override.length) + arch_flags = [ arch_override != "x86_64" ? "-m32" : "-m64" ]; + else + { + // Don't use Optlink by default on Windows + version (Windows) { + const is64bit = isWow64(); + if (!is64bit.isNull) + arch_flags = [ is64bit.get ? "-m64" : "-m32" ]; + } } - settings.addDFlags(arch_flags); - return probePlatform( + BuildPlatform bp = probePlatform( compiler_binary, arch_flags ~ ["-quiet", "-c", "-o-", "-v"], arch_override ); + + /// Replace archticture string in `bp.archtiecture` + void replaceArch(const string from, const string to) + { + const idx = bp.architecture.countUntil(from); + if (idx != -1) + bp.architecture[idx] = to; + } + + // DMD 2.099 changed the default for -m32 from OMF to MsCOFF + const m32IsCoff = bp.frontendVersion >= 2_099; + + switch (arch_override) { + default: throw new UnsupportedArchitectureException(arch_override); + case "": break; + case "x86": arch_flags = ["-m32"]; break; + case "x86_64": arch_flags = ["-m64"]; break; + + case "x86_omf": + if (m32IsCoff) + { + arch_flags = [ "-m32omf" ]; + replaceArch("x86_mscoff", "x86_omf"); // Probe used the wrong default + } + else // -m32 is OMF + { + arch_flags = [ "-m32" ]; + } + break; + + case "x86_mscoff": + if (m32IsCoff) + { + arch_flags = [ "-m32" ]; + } + else // -m32 is OMF + { + arch_flags = [ "-m32mscoff" ]; + replaceArch("x86_omf", "x86_mscoff"); // Probe used the wrong default + } + break; + } + settings.addDFlags(arch_flags); + + return bp; } version (Windows) version (DigitalMars) unittest { @@ -167,15 +208,17 @@ } version (LDC) unittest { + import std.conv : to; + BuildSettings settings; auto compiler = new DMDCompiler; auto bp = compiler.determinePlatform(settings, "ldmd2", "x86"); - assert(bp.architecture.canFind("x86")); - assert(!bp.architecture.canFind("x86_omf")); + assert(bp.architecture.canFind("x86"), bp.architecture.to!string); + assert(!bp.architecture.canFind("x86_omf"), bp.architecture.to!string); bp = compiler.determinePlatform(settings, "ldmd2", ""); - version (X86) assert(bp.architecture.canFind("x86")); - version (X86_64) assert(bp.architecture.canFind("x86_64")); - assert(!bp.architecture.canFind("x86_omf")); + version (X86) assert(bp.architecture.canFind("x86"), bp.architecture.to!string); + version (X86_64) assert(bp.architecture.canFind("x86_64"), bp.architecture.to!string); + assert(!bp.architecture.canFind("x86_omf"), bp.architecture.to!string); } void prepareBuildSettings(ref BuildSettings settings, const scope ref BuildPlatform platform, diff --git a/source/dub/compilers/gdc.d b/source/dub/compilers/gdc.d index f0fd762..04ed9dd 100644 --- a/source/dub/compilers/gdc.d +++ b/source/dub/compilers/gdc.d @@ -27,15 +27,15 @@ tuple(BuildOption.releaseMode, ["-frelease"]), tuple(BuildOption.coverage, ["-fprofile-arcs", "-ftest-coverage"]), tuple(BuildOption.debugInfo, ["-g"]), - tuple(BuildOption.debugInfoC, ["-g", "-fdebug-c"]), + tuple(BuildOption.debugInfoC, ["-g"]), //tuple(BuildOption.alwaysStackFrame, ["-X"]), //tuple(BuildOption.stackStomping, ["-X"]), tuple(BuildOption.inline, ["-finline-functions"]), tuple(BuildOption.noBoundsCheck, ["-fno-bounds-check"]), - tuple(BuildOption.optimize, ["-O3"]), + tuple(BuildOption.optimize, ["-O2"]), tuple(BuildOption.profile, ["-pg"]), tuple(BuildOption.unittests, ["-funittest"]), - tuple(BuildOption.verbose, ["-fd-verbose"]), + tuple(BuildOption.verbose, ["-v"]), tuple(BuildOption.ignoreUnknownPragmas, ["-fignore-unknown-pragmas"]), tuple(BuildOption.syntaxOnly, ["-fsyntax-only"]), tuple(BuildOption.warnings, ["-Wall"]), @@ -45,9 +45,10 @@ tuple(BuildOption.deprecationErrors, ["-Werror", "-Wdeprecated"]), tuple(BuildOption.property, ["-fproperty"]), //tuple(BuildOption.profileGC, ["-?"]), + tuple(BuildOption.betterC, ["-fno-druntime"]), tuple(BuildOption._docs, ["-fdoc-dir=docs"]), - tuple(BuildOption._ddox, ["-fXf=docs.json", "-fdoc-file=__dummy.html"]), + tuple(BuildOption._ddox, ["-Xfdocs.json", "-fdoc-file=__dummy.html"]), ]; @property string name() const { return "gdc"; } diff --git a/source/dub/description.d b/source/dub/description.d index 2953107..f26563b 100644 --- a/source/dub/description.d +++ b/source/dub/description.d @@ -84,18 +84,19 @@ string[] dflags; /// Flags passed to the D compiler string[] lflags; /// Flags passed to the linker string[] libs; /// Librariy names to link against (typically using "-l") + string[] injectSourceFiles; /// Files that should be injected when this package is dependend upon by a binary image. string[] copyFiles; /// Files to copy to the target directory string[] extraDependencyFiles; /// Files to check for rebuild dub project string[] versions; /// D version identifiers to set string[] debugVersions; /// D debug version identifiers to set string[] importPaths; string[] stringImportPaths; - string[] preGenerateCommands; /// commands executed before creating the description - string[] postGenerateCommands; /// commands executed after creating the description - string[] preBuildCommands; /// Commands to execute prior to every build - string[] postBuildCommands; /// Commands to execute after every build - string[] preRunCommands; /// Commands to execute prior to every run - string[] postRunCommands; /// Commands to execute after every run + string[] preGenerateCommands; /// Commands executed before creating the description, with variables not substituted. + string[] postGenerateCommands; /// Commands executed after creating the description, with variables not substituted. + string[] preBuildCommands; /// Commands to execute prior to every build, with variables not substituted. + string[] postBuildCommands; /// Commands to execute after every build, with variables not substituted. + string[] preRunCommands; /// Commands to execute prior to every run, with variables not substituted. + string[] postRunCommands; /// Commands to execute after every run, with variables not substituted. string[string] environments; string[string] buildEnvironments; string[string] runEnvironments; diff --git a/source/dub/dub.d b/source/dub/dub.d index 1b71bd7..0bc56eb 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -290,7 +290,15 @@ m_dirs.temp = NativePath(tempDir); m_config = new DubConfig(jsonFromFile(m_dirs.systemSettings ~ "settings.json", true), m_config); - m_config = new DubConfig(jsonFromFile(NativePath(thisExePath).parentPath ~ "../etc/dub/settings.json", true), m_config); + + auto dubFolderPath = NativePath(thisExePath).parentPath; + m_config = new DubConfig(jsonFromFile(dubFolderPath ~ "../etc/dub/settings.json", true), m_config); + version (Posix) { + if (dubFolderPath.absolute && dubFolderPath.startsWith(NativePath("usr"))) { + m_config = new DubConfig(jsonFromFile(NativePath("/etc/dub/settings.json"), true), m_config); + } + } + m_config = new DubConfig(jsonFromFile(m_dirs.userSettings ~ "settings.json", true), m_config); if (!root_path.empty) diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 4251789..e089e39 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -146,6 +146,8 @@ private bool buildTarget(GeneratorSettings settings, BuildSettings buildsettings, in Package pack, string config, in Package[] packages, in NativePath[] additional_dep_files, out NativePath target_path) { + import std.path : absolutePath; + auto cwd = NativePath(getcwd()); bool generate_binary = !(buildsettings.options & BuildOption.syntaxOnly); @@ -175,8 +177,8 @@ // run post-build commands if (!cached && buildsettings.postBuildCommands.length) { logInfo("Running post-build commands..."); - runBuildCommands(buildsettings.postBuildCommands, pack, m_project, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.postBuildEnvironments]); + runBuildCommands(CommandType.postBuild, buildsettings.postBuildCommands, pack, m_project, settings, buildsettings, + [["DUB_BUILD_PATH" : target_path.parentPath.toNativeString.absolutePath]]); } return cached; @@ -214,8 +216,7 @@ if( buildsettings.preBuildCommands.length ){ logInfo("Running pre-build commands..."); - runBuildCommands(buildsettings.preBuildCommands, pack, m_project, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preBuildEnvironments]); + runBuildCommands(CommandType.preBuild, buildsettings.preBuildCommands, pack, m_project, settings, buildsettings); } // override target path @@ -335,8 +336,7 @@ if( buildsettings.preBuildCommands.length ){ logInfo("Running pre-build commands..."); - runBuildCommands(buildsettings.preBuildCommands, pack, m_project, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preBuildEnvironments]); + runBuildCommands(CommandType.preBuild, buildsettings.preBuildCommands, pack, m_project, settings, buildsettings); } buildWithCompiler(settings, buildsettings); @@ -580,8 +580,7 @@ { if (buildsettings.preRunCommands.length) { logInfo("Running pre-run commands..."); - runBuildCommands(buildsettings.preRunCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.runEnvironments, buildsettings.preRunEnvironments]); + runBuildCommands(CommandType.preRun, buildsettings.preRunCommands, pack, proj, settings, buildsettings); } } @@ -590,8 +589,7 @@ { if (buildsettings.postRunCommands.length) { logInfo("Running post-run commands..."); - runBuildCommands(buildsettings.postRunCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.runEnvironments, buildsettings.postRunEnvironments]); + runBuildCommands(CommandType.postRun, buildsettings.postRunCommands, pack, proj, settings, buildsettings); } } diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index 2c87ee9..38ea369 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -169,8 +169,10 @@ prepareGeneration(pack, m_project, settings, bs); // Regenerate buildSettings.sourceFiles - if (bs.preGenerateCommands.length) - bs = makeBuildSettings(pack, targets[pack.name].buildSettings); + if (bs.preGenerateCommands.length) { + auto newSettings = pack.getBuildSettings(settings.platform, configs[pack.name]); + bs = makeBuildSettings(pack, newSettings); + } targets[pack.name].buildSettings = bs; } configurePackages(m_project.rootPackage, targets, settings); @@ -234,8 +236,12 @@ compatible. This also transports all Have_dependency_xyz version identifiers to `rootPackage`. - 4. Filter unused versions and debugVersions from all targets. The - filters have previously been upwards inherited (3.) so that versions + 4. Merge injectSourceFiles from dependencies into their dependents. + This is based upon binary images and will transcend direct relationships + including shared libraries. + + 5. Filter unused versions and debugVersions from all targets. The + filters have previously been upwards inherited (3. and 4.) so that versions used in a dependency are also applied to all dependents. Note: The upwards inheritance is done at last so that siblings do not @@ -252,15 +258,21 @@ auto roottarget = &targets[rootPackage.name]; // 0. do shallow configuration (not including dependencies) of all packages - TargetType determineTargetType(const ref TargetInfo ti) + TargetType determineTargetType(const ref TargetInfo ti, const ref GeneratorSettings genSettings) { TargetType tt = ti.buildSettings.targetType; if (ti.pack is rootPackage) { if (tt == TargetType.autodetect || tt == TargetType.library) tt = TargetType.staticLibrary; } else { if (tt == TargetType.autodetect || tt == TargetType.library) tt = genSettings.combined ? TargetType.sourceLibrary : TargetType.staticLibrary; - else if (tt == TargetType.dynamicLibrary) { - logWarn("Dynamic libraries are not yet supported as dependencies - building as static library."); + else if (genSettings.platform.architecture.canFind("x86_omf") && tt == TargetType.dynamicLibrary) { + // Unfortunately we cannot remove this check for OMF targets, + // due to Optlink not producing shared libraries without a lot of user intervention. + // For other targets, say MSVC it'll do the right thing for the most part, + // export is still a problem as of this writing, which means static libraries cannot have linking to them removed. + // But that is for the most part up to the developer, to get it working correctly. + + logWarn("Dynamic libraries are not yet supported as dependencies for Windows target OMF - building as static library."); tt = TargetType.staticLibrary; } } @@ -279,7 +291,7 @@ { auto bs = &ti.buildSettings; // determine the actual target type - bs.targetType = determineTargetType(ti); + bs.targetType = determineTargetType(ti, genSettings); switch (bs.targetType) { @@ -412,6 +424,11 @@ } // 3. upwards inherit full build configurations (import paths, versions, debugVersions, versionFilters, importPaths, ...) + + // We do a check for if any dependency uses final binary injection source files, + // otherwise can ignore that bit of workload entirely + bool skipFinalBinaryMerging = true; + void configureDependents(ref TargetInfo ti, TargetInfo[string] targets, size_t level = 0) { // use `visited` here as pkgs cannot depend on themselves @@ -423,19 +440,67 @@ // embedded non-binary dependencies foreach (deppack; ti.packages[1 .. $]) ti.buildSettings.add(targets[deppack.name].buildSettings); + // binary dependencies foreach (depname; ti.dependencies) { auto pdepti = &targets[depname]; + configureDependents(*pdepti, targets, level + 1); mergeFromDependency(pdepti.buildSettings, ti.buildSettings, genSettings.platform); + + if (!pdepti.buildSettings.injectSourceFiles.empty) + skipFinalBinaryMerging = false; } } configureDependents(*roottarget, targets); visited.clear(); - // 4. Filter applicable version and debug version identifiers + // 4. As an extension to configureDependents we need to copy any injectSourceFiles + // in our dependencies (ignoring targetType) + void configureDependentsFinalImages(ref TargetInfo ti, TargetInfo[string] targets, ref TargetInfo finalBinaryTarget, size_t level = 0) + { + // use `visited` here as pkgs cannot depend on themselves + if (ti.pack in visited) + return; + visited[ti.pack] = typeof(visited[ti.pack]).init; + + logDiagnostic("%sConfiguring dependent %s, deps:%(%s, %) for injectSourceFiles", ' '.repeat(2 * level), ti.pack.name, ti.dependencies); + + foreach (depname; ti.dependencies) + { + auto pdepti = &targets[depname]; + + if (!pdepti.buildSettings.injectSourceFiles.empty) + finalBinaryTarget.buildSettings.addSourceFiles(pdepti.buildSettings.injectSourceFiles); + + configureDependentsFinalImages(*pdepti, targets, finalBinaryTarget, level + 1); + } + } + + if (!skipFinalBinaryMerging) + { + foreach (ref target; targets.byValue) + { + switch (target.buildSettings.targetType) + { + case TargetType.executable: + case TargetType.dynamicLibrary: + configureDependentsFinalImages(target, targets, target); + + // We need to clear visited for each target that is executable dynamicLibrary + // due to this process needing to be recursive based upon the final binary targets. + visited.clear(); + break; + + default: + break; + } + } + } + + // 5. Filter applicable version and debug version identifiers if (genSettings.filterVersions) { foreach (name, ref ti; targets) @@ -454,7 +519,7 @@ } } - // 5. override string import files in dependencies + // 6. override string import files in dependencies static void overrideStringImports(ref TargetInfo target, ref TargetInfo parent, TargetInfo[string] targets, string[] overrides) { @@ -505,7 +570,7 @@ overrideStringImports(targets[depname], *roottarget, targets, roottarget.buildSettings.stringImportFiles); - // 6. downwards inherits dependency build settings + // 7. downwards inherits dependency build settings static void applyForcedSettings(const scope ref TargetInfo ti, TargetInfo[string] targets, BuildSettings[string] dependBS, size_t level = 0) { @@ -669,6 +734,7 @@ parent.addDebugVersionFilters(child.debugVersionFilters); parent.addImportPaths(child.importPaths); parent.addStringImportPaths(child.stringImportPaths); + parent.addInjectSourceFiles(child.injectSourceFiles); // linking of static libraries is done by parent if (child.targetType == TargetType.staticLibrary) { parent.addSourceFiles(child.sourceFiles.filter!(f => isLinkerFile(platform, f)).array); @@ -826,8 +892,7 @@ { if (buildsettings.preGenerateCommands.length && !isRecursiveInvocation(pack.name)) { logInfo("Running pre-generate commands for %s...", pack.name); - runBuildCommands(buildsettings.preGenerateCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preGenerateEnvironments]); + runBuildCommands(CommandType.preGenerate, buildsettings.preGenerateCommands, pack, proj, settings, buildsettings); } } @@ -839,8 +904,7 @@ { if (buildsettings.postGenerateCommands.length && !isRecursiveInvocation(pack.name)) { logInfo("Running post-generate commands for %s...", pack.name); - runBuildCommands(buildsettings.postGenerateCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.postGenerateEnvironments]); + runBuildCommands(CommandType.postGenerate, buildsettings.postGenerateCommands, pack, proj, settings, buildsettings); } if (generate_binary) { @@ -907,29 +971,44 @@ command execution loops. The latter could otherwise happen when a command runs "dub describe" or similar functionality. */ -void runBuildCommands(in string[] commands, in Package pack, in Project proj, +void runBuildCommands(CommandType type, in string[] commands, in Package pack, in Project proj, in GeneratorSettings settings, in BuildSettings build_settings, in string[string][] extraVars = null) { - import dub.internal.utils : getDUBExePath, runCommands; + import dub.internal.utils : runCommands; + + auto env = makeCommandEnvironmentVariables(type, pack, proj, settings, build_settings, extraVars); + auto sub_commands = processVars(proj, pack, settings, commands, false, env); + + auto depNames = proj.dependencies.map!((a) => a.name).array(); + storeRecursiveInvokations(env, proj.rootPackage.name ~ depNames); + + runCommands(sub_commands, env.collapseEnv, pack.path().toString()); +} + +const(string[string])[] makeCommandEnvironmentVariables(CommandType type, + in Package pack, in Project proj, in GeneratorSettings settings, + in BuildSettings build_settings, in string[string][] extraVars = null) +{ + import dub.internal.utils : getDUBExePath; import std.conv : to, text; import std.process : environment, escapeShellFileName; - string[string] env = environment.toAA(); + string[string] env; // TODO: do more elaborate things here // TODO: escape/quote individual items appropriately - env["VERSIONS"] = join(cast(string[])build_settings.versions," "); - env["LIBS"] = join(cast(string[])build_settings.libs," "); - env["SOURCE_FILES"] = join(cast(string[])build_settings.sourceFiles," "); - env["IMPORT_PATHS"] = join(cast(string[])build_settings.importPaths," "); - env["STRING_IMPORT_PATHS"] = join(cast(string[])build_settings.stringImportPaths," "); + env["VERSIONS"] = join(build_settings.versions, " "); + env["LIBS"] = join(build_settings.libs, " "); + env["SOURCE_FILES"] = join(build_settings.sourceFiles, " "); + env["IMPORT_PATHS"] = join(build_settings.importPaths, " "); + env["STRING_IMPORT_PATHS"] = join(build_settings.stringImportPaths, " "); env["DC"] = settings.platform.compilerBinary; env["DC_BASE"] = settings.platform.compiler; env["D_FRONTEND_VER"] = to!string(settings.platform.frontendVersion); env["DUB_EXE"] = getDUBExePath(settings.platform.compilerBinary); - env["DUB_PLATFORM"] = join(cast(string[])settings.platform.platform," "); - env["DUB_ARCH"] = join(cast(string[])settings.platform.architecture," "); + env["DUB_PLATFORM"] = join(settings.platform.platform, " "); + env["DUB_ARCH"] = join(settings.platform.architecture, " "); env["DUB_TARGET_TYPE"] = to!string(build_settings.targetType); env["DUB_TARGET_PATH"] = build_settings.targetPath; @@ -962,15 +1041,49 @@ env["DUB_ROOT_PACKAGE_TARGET_TYPE"] = to!string(rootPackageBuildSettings.targetType); env["DUB_ROOT_PACKAGE_TARGET_PATH"] = rootPackageBuildSettings.targetPath; env["DUB_ROOT_PACKAGE_TARGET_NAME"] = rootPackageBuildSettings.targetName; - - foreach (aa; extraVars) { - foreach (k, v; aa) - env[k] = v; + + const(string[string])[] typeEnvVars; + with (build_settings) final switch (type) + { + // pre/postGenerate don't have generateEnvironments, but reuse buildEnvironments + case CommandType.preGenerate: typeEnvVars = [environments, buildEnvironments, preGenerateEnvironments]; break; + case CommandType.postGenerate: typeEnvVars = [environments, buildEnvironments, postGenerateEnvironments]; break; + case CommandType.preBuild: typeEnvVars = [environments, buildEnvironments, preBuildEnvironments]; break; + case CommandType.postBuild: typeEnvVars = [environments, buildEnvironments, postBuildEnvironments]; break; + case CommandType.preRun: typeEnvVars = [environments, runEnvironments, preRunEnvironments]; break; + case CommandType.postRun: typeEnvVars = [environments, runEnvironments, postRunEnvironments]; break; } - auto depNames = proj.dependencies.map!((a) => a.name).array(); - storeRecursiveInvokations(env, proj.rootPackage.name ~ depNames); - runCommands(commands, env, pack.path().toString()); + return [environment.toAA()] ~ env ~ typeEnvVars ~ extraVars; +} + +string[string] collapseEnv(in string[string][] envs) +{ + string[string] ret; + foreach (subEnv; envs) + { + foreach (k, v; subEnv) + ret[k] = v; + } + return ret; +} + +/// Type to specify where CLI commands that need to be run came from. Needed for +/// proper substitution with support for the different environments. +enum CommandType +{ + /// Defined in the preGenerateCommands setting + preGenerate, + /// Defined in the postGenerateCommands setting + postGenerate, + /// Defined in the preBuildCommands setting + preBuild, + /// Defined in the postBuildCommands setting + postBuild, + /// Defined in the preRunCommands setting + preRun, + /// Defined in the postRunCommands setting + postRun } private bool isRecursiveInvocation(string pack) @@ -979,20 +1092,70 @@ import std.process : environment; return environment - .get("DUB_PACKAGES_USED", "") - .splitter(",") - .canFind(pack); + .get("DUB_PACKAGES_USED", "") + .splitter(",") + .canFind(pack); } -private void storeRecursiveInvokations(string[string] env, string[] packs) +private void storeRecursiveInvokations(ref const(string[string])[] env, string[] packs) { import std.algorithm : canFind, splitter; import std.range : chain; import std.process : environment; - env["DUB_PACKAGES_USED"] = environment - .get("DUB_PACKAGES_USED", "") - .splitter(",") - .chain(packs) - .join(","); + env ~= [ + "DUB_PACKAGES_USED": environment + .get("DUB_PACKAGES_USED", "") + .splitter(",") + .chain(packs) + .join(",") + ]; +} + +version(Posix) { + // https://github.com/dlang/dub/issues/2238 + unittest { + import dub.internal.vibecompat.data.json : parseJsonString; + import dub.compilers.gdc : GDCCompiler; + import std.algorithm : canFind; + import std.path : absolutePath; + import std.file : mkdirRecurse, rmdirRecurse, write; + + mkdirRecurse("dubtest/preGen/source"); + write("dubtest/preGen/source/foo.d", ""); + scope(exit) rmdirRecurse("dubtest"); + + auto desc = parseJsonString(`{"name": "test", "targetType": "library", "preGenerateCommands": ["touch $PACKAGE_DIR/source/bar.d"]}`); + auto pack = new Package(desc, NativePath("dubtest/preGen".absolutePath)); + auto pman = new PackageManager(pack.path, NativePath("/tmp/foo/"), NativePath("/tmp/foo/"), false); + auto prj = new Project(pman, pack); + + final static class TestCompiler : GDCCompiler { + override void invoke(in BuildSettings settings, in BuildPlatform platform, void delegate(int, string) output_callback) { + assert(false); + } + override void invokeLinker(in BuildSettings settings, in BuildPlatform platform, string[] objects, void delegate(int, string) output_callback) { + assert(false); + } + } + + GeneratorSettings settings; + settings.compiler = new TestCompiler; + settings.buildType = "debug"; + + final static class TestGenerator : ProjectGenerator { + this(Project project) { + super(project); + } + + override void generateTargets(GeneratorSettings settings, in TargetInfo[string] targets) { + import std.conv : text; + const sourceFiles = targets["test"].buildSettings.sourceFiles; + assert(sourceFiles.canFind("dubtest/preGen/source/bar.d".absolutePath), sourceFiles.text); + } + } + + auto gen = new TestGenerator(prj); + gen.generate(settings); + } } diff --git a/source/dub/internal/vibecompat/data/json.d b/source/dub/internal/vibecompat/data/json.d index 0000184..b55166a 100644 --- a/source/dub/internal/vibecompat/data/json.d +++ b/source/dub/internal/vibecompat/data/json.d @@ -55,12 +55,12 @@ // the issues. void*[2] m_data; ref inout(T) getDataAs(T)() inout { static assert(T.sizeof <= m_data.sizeof); return *cast(inout(T)*)m_data.ptr; } - @property ref inout(long) m_int() inout { return getDataAs!long(); } - @property ref inout(double) m_float() inout { return getDataAs!double(); } - @property ref inout(bool) m_bool() inout { return getDataAs!bool(); } - @property ref inout(string) m_string() inout { return getDataAs!string(); } - @property ref inout(Json[string]) m_object() inout { return getDataAs!(Json[string])(); } - @property ref inout(Json[]) m_array() inout { return getDataAs!(Json[])(); } + @property ref inout(long) m_int() inout return { return getDataAs!long(); } + @property ref inout(double) m_float() inout return { return getDataAs!double(); } + @property ref inout(bool) m_bool() inout return { return getDataAs!bool(); } + @property ref inout(string) m_string() inout return { return getDataAs!string(); } + @property ref inout(Json[string]) m_object() inout return { return getDataAs!(Json[string])(); } + @property ref inout(Json[]) m_array() inout return { return getDataAs!(Json[])(); } Type m_type = Type.undefined; diff --git a/source/dub/package_.d b/source/dub/package_.d index cf49f92..98b463b 100644 --- a/source/dub/package_.d +++ b/source/dub/package_.d @@ -624,6 +624,7 @@ ret.dflags = bs.dflags; ret.lflags = bs.lflags; ret.libs = bs.libs; + ret.injectSourceFiles = bs.injectSourceFiles; ret.copyFiles = bs.copyFiles; ret.versions = bs.versions; ret.debugVersions = bs.debugVersions; diff --git a/source/dub/packagesuppliers/filesystem.d b/source/dub/packagesuppliers/filesystem.d index e271510..9cfb55e 100644 --- a/source/dub/packagesuppliers/filesystem.d +++ b/source/dub/packagesuppliers/filesystem.d @@ -52,6 +52,7 @@ { import std.array : split; import std.path : stripExtension; + import std.algorithm : startsWith, endsWith; import dub.internal.utils : packageInfoFileFromZip; import dub.recipe.io : parsePackageRecipe; import dub.recipe.json : toJson; @@ -61,7 +62,10 @@ string packageFileContent = packageInfoFileFromZip(filePath, packageFileName); auto recipe = parsePackageRecipe(packageFileContent, packageFileName); Json json = toJson(recipe); - json["version"] = filePath.toNativeString().split("-")[$-1].stripExtension(); + auto basename = filePath.head.name; + enforce(basename.endsWith(".zip"), "Malformed package filename: " ~ filePath.toNativeString); + enforce(basename.startsWith(packageId), "Malformed package filename: " ~ filePath.toNativeString); + json["version"] = basename[packageId.length + 1 .. $-4]; return json; } diff --git a/source/dub/project.d b/source/dub/project.d index 026697d..e7361d0 100644 --- a/source/dub/project.d +++ b/source/dub/project.d @@ -751,24 +751,24 @@ return ret; } - private string[] listBuildSetting(string attributeName)(BuildPlatform platform, + private string[] listBuildSetting(string attributeName)(ref GeneratorSettings settings, string config, ProjectDescription projectDescription, Compiler compiler, bool disableEscaping) { - return listBuildSetting!attributeName(platform, getPackageConfigs(platform, config), + return listBuildSetting!attributeName(settings, getPackageConfigs(settings.platform, config), projectDescription, compiler, disableEscaping); } - private string[] listBuildSetting(string attributeName)(BuildPlatform platform, + private string[] listBuildSetting(string attributeName)(ref GeneratorSettings settings, string[string] configs, ProjectDescription projectDescription, Compiler compiler, bool disableEscaping) { if (compiler) - return formatBuildSettingCompiler!attributeName(platform, configs, projectDescription, compiler, disableEscaping); + return formatBuildSettingCompiler!attributeName(settings, configs, projectDescription, compiler, disableEscaping); else - return formatBuildSettingPlain!attributeName(platform, configs, projectDescription); + return formatBuildSettingPlain!attributeName(settings, configs, projectDescription); } // Output a build setting formatted for a compiler - private string[] formatBuildSettingCompiler(string attributeName)(BuildPlatform platform, + private string[] formatBuildSettingCompiler(string attributeName)(ref GeneratorSettings settings, string[string] configs, ProjectDescription projectDescription, Compiler compiler, bool disableEscaping) { import std.process : escapeShellFileName; @@ -786,11 +786,12 @@ case "linkerFiles": case "mainSourceFile": case "importFiles": - values = formatBuildSettingPlain!attributeName(platform, configs, projectDescription); + values = formatBuildSettingPlain!attributeName(settings, configs, projectDescription); break; case "lflags": case "sourceFiles": + case "injectSourceFiles": case "versions": case "debugVersions": case "importPaths": @@ -806,7 +807,7 @@ else static if (attributeName == "stringImportPaths") bs.stringImportPaths = bs.stringImportPaths.map!(ensureTrailingSlash).array(); - compiler.prepareBuildSettings(bs, platform, BuildSetting.all & ~to!BuildSetting(attributeName)); + compiler.prepareBuildSettings(bs, settings.platform, BuildSetting.all & ~to!BuildSetting(attributeName)); values = bs.dflags; break; @@ -817,7 +818,7 @@ bs.sourceFiles = null; bs.targetType = TargetType.none; // Force Compiler to NOT omit dependency libs when package is a library. - compiler.prepareBuildSettings(bs, platform, BuildSetting.all & ~to!BuildSetting(attributeName)); + compiler.prepareBuildSettings(bs, settings.platform, BuildSetting.all & ~to!BuildSetting(attributeName)); if (bs.lflags) values = compiler.lflagsToDFlags( bs.lflags ); @@ -838,6 +839,7 @@ { case "mainSourceFile": case "linkerFiles": + case "injectSourceFiles": case "copyFiles": case "importFiles": case "stringImportFiles": @@ -855,7 +857,7 @@ } // Output a build setting without formatting for any particular compiler - private string[] formatBuildSettingPlain(string attributeName)(BuildPlatform platform, string[string] configs, ProjectDescription projectDescription) + private string[] formatBuildSettingPlain(string attributeName)(ref GeneratorSettings settings, string[string] configs, ProjectDescription projectDescription) { import std.path : buildNormalizedPath, dirSeparator; import std.range : only; @@ -872,6 +874,12 @@ auto targetDescription = projectDescription.lookupTarget(projectDescription.rootPackage); auto buildSettings = targetDescription.buildSettings; + string[] substituteCommands(Package pack, string[] commands, CommandType type) + { + auto env = makeCommandEnvironmentVariables(type, pack, this, settings, buildSettings); + return processVars(this, pack, settings, commands, false, env); + } + // Return any BuildSetting member attributeName as a range of strings. Don't attempt to fixup values. // allowEmptyString: When the value is a string (as opposed to string[]), // is empty string an actual permitted value instead of @@ -879,7 +887,9 @@ auto getRawBuildSetting(Package pack, bool allowEmptyString) { auto value = __traits(getMember, buildSettings, attributeName); - static if( is(typeof(value) == string[]) ) + static if( attributeName.endsWith("Commands") ) + return substituteCommands(pack, value, mixin("CommandType.", attributeName[0 .. $ - "Commands".length])); + else static if( is(typeof(value) == string[]) ) return value; else static if( is(typeof(value) == string) ) { @@ -918,7 +928,8 @@ enum isRelativeFile = attributeName == "sourceFiles" || attributeName == "linkerFiles" || attributeName == "importFiles" || attributeName == "stringImportFiles" || - attributeName == "copyFiles" || attributeName == "mainSourceFile"; + attributeName == "copyFiles" || attributeName == "mainSourceFile" || + attributeName == "injectSourceFiles"; // For these, empty string means "main project directory", not "missing value" enum allowEmptyString = @@ -959,7 +970,7 @@ // The "compiler" arg is for choosing which compiler the output should be formatted for, // or null to imply "list" format. - private string[] listBuildSetting(BuildPlatform platform, string[string] configs, + private string[] listBuildSetting(ref GeneratorSettings settings, string[string] configs, ProjectDescription projectDescription, string requestedData, Compiler compiler, bool disableEscaping) { // Certain data cannot be formatter for a compiler @@ -978,6 +989,8 @@ case "post-generate-commands": case "pre-build-commands": case "post-build-commands": + case "pre-run-commands": + case "post-run-commands": case "environments": case "build-environments": case "run-environments": @@ -999,7 +1012,7 @@ } import std.typetuple : TypeTuple; - auto args = TypeTuple!(platform, configs, projectDescription, compiler, disableEscaping); + auto args = TypeTuple!(settings, configs, projectDescription, compiler, disableEscaping); switch (requestedData) { case "target-type": return listBuildSetting!"targetType"(args); @@ -1012,6 +1025,7 @@ case "libs": return listBuildSetting!"libs"(args); case "linker-files": return listBuildSetting!"linkerFiles"(args); case "source-files": return listBuildSetting!"sourceFiles"(args); + case "inject-source-files": return listBuildSetting!"injectSourceFiles"(args); case "copy-files": return listBuildSetting!"copyFiles"(args); case "extra-dependency-files": return listBuildSetting!"extraDependencyFiles"(args); case "versions": return listBuildSetting!"versions"(args); @@ -1084,7 +1098,7 @@ } auto result = requestedData - .map!(dataName => listBuildSetting(settings.platform, configs, projectDescription, dataName, compiler, no_escape)); + .map!(dataName => listBuildSetting(settings, configs, projectDescription, dataName, compiler, no_escape)); final switch (list_type) with (ListBuildSettingsFormat) { case list: return result.map!(l => l.join("\n")).array(); @@ -1206,13 +1220,6 @@ dst.addPostRunEnvironments(processVerEnvs(settings.postRunEnvironments, gsettings.buildSettings.postRunEnvironments)); auto buildEnvs = [dst.environments, dst.buildEnvironments]; - auto runEnvs = [dst.environments, dst.runEnvironments]; - auto preGenEnvs = [dst.environments, dst.preGenerateEnvironments]; - auto postGenEnvs = [dst.environments, dst.postGenerateEnvironments]; - auto preBuildEnvs = buildEnvs ~ [dst.preBuildEnvironments]; - auto postBuildEnvs = buildEnvs ~ [dst.postBuildEnvironments]; - auto preRunEnvs = runEnvs ~ [dst.preRunEnvironments]; - auto postRunEnvs = runEnvs ~ [dst.postRunEnvironments]; dst.addDFlags(processVars(project, pack, gsettings, settings.dflags, false, buildEnvs)); dst.addLFlags(processVars(project, pack, gsettings, settings.lflags, false, buildEnvs)); @@ -1220,6 +1227,7 @@ dst.addSourceFiles(processVars!true(project, pack, gsettings, settings.sourceFiles, true, buildEnvs)); dst.addImportFiles(processVars(project, pack, gsettings, settings.importFiles, true, buildEnvs)); dst.addStringImportFiles(processVars(project, pack, gsettings, settings.stringImportFiles, true, buildEnvs)); + dst.addInjectSourceFiles(processVars!true(project, pack, gsettings, settings.injectSourceFiles, true, buildEnvs)); dst.addCopyFiles(processVars(project, pack, gsettings, settings.copyFiles, true, buildEnvs)); dst.addExtraDependencyFiles(processVars(project, pack, gsettings, settings.extraDependencyFiles, true, buildEnvs)); dst.addVersions(processVars(project, pack, gsettings, settings.versions, false, buildEnvs)); @@ -1228,15 +1236,17 @@ dst.addDebugVersionFilters(processVars(project, pack, gsettings, settings.debugVersionFilters, false, buildEnvs)); dst.addImportPaths(processVars(project, pack, gsettings, settings.importPaths, true, buildEnvs)); dst.addStringImportPaths(processVars(project, pack, gsettings, settings.stringImportPaths, true, buildEnvs)); - dst.addPreGenerateCommands(processVars(project, pack, gsettings, settings.preGenerateCommands, false, preGenEnvs)); - dst.addPostGenerateCommands(processVars(project, pack, gsettings, settings.postGenerateCommands, false, postGenEnvs)); - dst.addPreBuildCommands(processVars(project, pack, gsettings, settings.preBuildCommands, false, preBuildEnvs)); - dst.addPostBuildCommands(processVars(project, pack, gsettings, settings.postBuildCommands, false, postBuildEnvs)); - dst.addPreRunCommands(processVars(project, pack, gsettings, settings.preRunCommands, false, preRunEnvs)); - dst.addPostRunCommands(processVars(project, pack, gsettings, settings.postRunCommands, false, postRunEnvs)); dst.addRequirements(settings.requirements); dst.addOptions(settings.options); + // commands are substituted in dub.generators.generator : runBuildCommands + dst.addPreGenerateCommands(settings.preGenerateCommands); + dst.addPostGenerateCommands(settings.postGenerateCommands); + dst.addPreBuildCommands(settings.preBuildCommands); + dst.addPostBuildCommands(settings.postBuildCommands); + dst.addPreRunCommands(settings.preRunCommands); + dst.addPostRunCommands(settings.postRunCommands); + if (include_target_settings) { dst.targetType = settings.targetType; dst.targetPath = processVars(settings.targetPath, project, pack, gsettings, true, buildEnvs); @@ -1248,13 +1258,13 @@ } } -private string[] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, string[] vars, bool are_paths = false, in string[string][] extraVers = null) +string[] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, in string[] vars, bool are_paths = false, in string[string][] extraVers = null) { auto ret = appender!(string[])(); processVars!glob(ret, project, pack, gsettings, vars, are_paths, extraVers); return ret.data; } -private void processVars(bool glob = false)(ref Appender!(string[]) dst, in Project project, in Package pack, in GeneratorSettings gsettings, string[] vars, bool are_paths = false, in string[string][] extraVers = null) +void processVars(bool glob = false)(ref Appender!(string[]) dst, in Project project, in Package pack, in GeneratorSettings gsettings, in string[] vars, bool are_paths = false, in string[string][] extraVers = null) { static if (glob) alias process = processVarsWithGlob!(Project, Package); @@ -1264,7 +1274,7 @@ dst.put(process(var, project, pack, gsettings, are_paths, extraVers)); } -private string processVars(Project, Package)(string var, in Project project, in Package pack, in GeneratorSettings gsettings, bool is_path, in string[string][] extraVers = null) +string processVars(Project, Package)(string var, in Project project, in Package pack, in GeneratorSettings gsettings, bool is_path, in string[string][] extraVers = null) { var = var.expandVars!(varName => getVariable(varName, project, pack, gsettings, extraVers)); if (!is_path) @@ -1275,13 +1285,13 @@ else return p.toNativeString(); } -private string[string] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, string[string] vars, in string[string][] extraVers = null) +string[string] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, in string[string] vars, in string[string][] extraVers = null) { string[string] ret; processVars!glob(ret, project, pack, gsettings, vars, extraVers); return ret; } -private void processVars(bool glob = false)(ref string[string] dst, in Project project, in Package pack, in GeneratorSettings gsettings, string[string] vars, in string[string][] extraVers) +void processVars(bool glob = false)(ref string[string] dst, in Project project, in Package pack, in GeneratorSettings gsettings, in string[string] vars, in string[string][] extraVers) { static if (glob) alias process = processVarsWithGlob!(Project, Package); diff --git a/source/dub/recipe/json.d b/source/dub/recipe/json.d index 0c5597b..c1d776d 100644 --- a/source/dub/recipe/json.d +++ b/source/dub/recipe/json.d @@ -221,6 +221,7 @@ case "sourcePaths": bs.sourcePaths[suffix] = deserializeJson!(string[])(value); break; case "sourcePath": bs.sourcePaths[suffix] ~= [value.get!string]; break; // deprecated case "excludedSourceFiles": bs.excludedSourceFiles[suffix] = deserializeJson!(string[])(value); break; + case "injectSourceFiles": bs.injectSourceFiles[suffix] = deserializeJson!(string[])(value); break; case "copyFiles": bs.copyFiles[suffix] = deserializeJson!(string[])(value); break; case "extraDependencyFiles": bs.extraDependencyFiles[suffix] = deserializeJson!(string[])(value); break; case "versions": bs.versions[suffix] = deserializeJson!(string[])(value); break; @@ -282,6 +283,7 @@ foreach (suffix, arr; bs.sourceFiles) ret["sourceFiles"~suffix] = serializeToJson(arr); foreach (suffix, arr; bs.sourcePaths) ret["sourcePaths"~suffix] = serializeToJson(arr); foreach (suffix, arr; bs.excludedSourceFiles) ret["excludedSourceFiles"~suffix] = serializeToJson(arr); + foreach (suffix, arr; bs.injectSourceFiles) ret["injectSourceFiles"~suffix] = serializeToJson(arr); foreach (suffix, arr; bs.copyFiles) ret["copyFiles"~suffix] = serializeToJson(arr); foreach (suffix, arr; bs.extraDependencyFiles) ret["extraDependencyFiles"~suffix] = serializeToJson(arr); foreach (suffix, arr; bs.versions) ret["versions"~suffix] = serializeToJson(arr); @@ -376,7 +378,7 @@ parseJson(rec1, jsonValue, null); PackageRecipe rec; parseJson(rec, rec1.toJson(), null); // verify that all fields are serialized properly - + assert(rec.name == "projectname"); assert(rec.buildSettings.environments == ["": ["Var1": "env"]]); assert(rec.buildSettings.buildEnvironments == ["": ["Var2": "buildEnv"]]); diff --git a/source/dub/recipe/packagerecipe.d b/source/dub/recipe/packagerecipe.d index f924c2a..e1e4663 100644 --- a/source/dub/recipe/packagerecipe.d +++ b/source/dub/recipe/packagerecipe.d @@ -190,6 +190,7 @@ string[][string] sourceFiles; string[][string] sourcePaths; string[][string] excludedSourceFiles; + string[][string] injectSourceFiles; string[][string] copyFiles; string[][string] extraDependencyFiles; string[][string] versions; @@ -298,6 +299,7 @@ getPlatformSetting!("libs", "addLibs")(dst, platform); getPlatformSetting!("sourceFiles", "addSourceFiles")(dst, platform); getPlatformSetting!("excludedSourceFiles", "removeSourceFiles")(dst, platform); + getPlatformSetting!("injectSourceFiles", "addInjectSourceFiles")(dst, platform); getPlatformSetting!("copyFiles", "addCopyFiles")(dst, platform); getPlatformSetting!("extraDependencyFiles", "addExtraDependencyFiles")(dst, platform); getPlatformSetting!("versions", "addVersions")(dst, platform); diff --git a/source/dub/recipe/sdl.d b/source/dub/recipe/sdl.d index 177eeb9..e005e11 100644 --- a/source/dub/recipe/sdl.d +++ b/source/dub/recipe/sdl.d @@ -153,6 +153,7 @@ case "sourcePaths": setting.parsePlatformStringArray(bs.sourcePaths); break; case "excludedSourceFiles": setting.parsePlatformStringArray(bs.excludedSourceFiles); break; case "mainSourceFile": bs.mainSourceFile = setting.stringTagValue; break; + case "injectSourceFiles": setting.parsePlatformStringArray(bs.injectSourceFiles); break; case "copyFiles": setting.parsePlatformStringArray(bs.copyFiles); break; case "extraDependencyFiles": setting.parsePlatformStringArray(bs.extraDependencyFiles); break; case "versions": setting.parsePlatformStringArray(bs.versions); break; @@ -285,6 +286,7 @@ foreach (suffix, arr; bs.sourceFiles) adda("sourceFiles", suffix, arr); foreach (suffix, arr; bs.sourcePaths) adda("sourcePaths", suffix, arr); foreach (suffix, arr; bs.excludedSourceFiles) adda("excludedSourceFiles", suffix, arr); + foreach (suffix, arr; bs.injectSourceFiles) adda("injectSourceFiles", suffix, arr); foreach (suffix, arr; bs.copyFiles) adda("copyFiles", suffix, arr); foreach (suffix, arr; bs.extraDependencyFiles) adda("extraDependencyFiles", suffix, arr); foreach (suffix, arr; bs.versions) adda("versions", suffix, arr); @@ -464,6 +466,7 @@ excludedSourceFiles "excluded1" "excluded2" excludedSourceFiles "excluded3" mainSourceFile "main source" +injectSourceFiles "finalbinarysourcefile.d" "extrafile" copyFiles "copy1" "copy2" copyFiles "copy3" extraDependencyFiles "extradepfile1" "extradepfile2" @@ -566,7 +569,8 @@ assert(rec.buildSettings.sourcePaths == ["": ["sourcepath1", "sourcepath2", "sourcepath3"]]); assert(rec.buildSettings.excludedSourceFiles == ["": ["excluded1", "excluded2", "excluded3"]]); assert(rec.buildSettings.mainSourceFile == "main source"); - assert(rec.buildSettings.copyFiles == ["": ["copy1", "copy2", "copy3"]]); + assert(rec.buildSettings.sourceFiles == ["": ["source1", "source2", "source3"]]); + assert(rec.buildSettings.injectSourceFiles == ["": ["finalbinarysourcefile.d", "extrafile"]]); assert(rec.buildSettings.extraDependencyFiles == ["": ["extradepfile1", "extradepfile2", "extradepfile3"]]); assert(rec.buildSettings.versions == ["": ["version1", "version2", "version3"]]); assert(rec.buildSettings.debugVersions == ["": ["debug1", "debug2", "debug3"]]); diff --git a/test/1-dynLib-simple/dub.json b/test/1-dynLib-simple/dub.json index c7747c3..71da415 100644 --- a/test/1-dynLib-simple/dub.json +++ b/test/1-dynLib-simple/dub.json @@ -1,4 +1,5 @@ { "name": "dynlib-simple", - "targetType": "dynamicLibrary" + "targetType": "dynamicLibrary", + "dflags-ldc": ["-link-defaultlib-shared"] } diff --git a/test/1-dynLib-simple/source/dynlib/app.d b/test/1-dynLib-simple/source/dynlib/app.d index 78fbd42..6676219 100644 --- a/test/1-dynLib-simple/source/dynlib/app.d +++ b/test/1-dynLib-simple/source/dynlib/app.d @@ -1,7 +1,7 @@ module dynlib.app; import std.stdio; -void entry() +export void entry() { writeln(__FUNCTION__); } diff --git a/test/2-dynLib-dep/dub.json b/test/2-dynLib-dep/dub.json index 393810d..1d3d5cf 100644 --- a/test/2-dynLib-dep/dub.json +++ b/test/2-dynLib-dep/dub.json @@ -2,5 +2,6 @@ "name": "dynlib-dep", "dependencies": { "dynlib-simple": { "path": "../1-dynLib-simple/" } - } + }, + "dflags-ldc": ["-link-defaultlib-shared"] } diff --git a/test/filesystem-version-with-buildinfo.sh b/test/filesystem-version-with-buildinfo.sh new file mode 100755 index 0000000..a9e24b1 --- /dev/null +++ b/test/filesystem-version-with-buildinfo.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +DIR=$(dirname "${BASH_SOURCE[0]}") + +. "$DIR"/common.sh + +${DUB} remove fs-json-dubpackage --non-interactive 2>/dev/null || true + +echo "Trying to get fs-json-dubpackage (1.0.7)" +${DUB} fetch fs-json-dubpackage@1.0.7 --skip-registry=all --registry=file://"$DIR"/filesystem-version-with-buildinfo + +if ! ${DUB} remove fs-json-dubpackage@1.0.7 2>/dev/null; then + die $LINENO 'DUB did not install package from file system.' +fi diff --git a/test/filesystem-version-with-buildinfo/.no_build b/test/filesystem-version-with-buildinfo/.no_build new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/filesystem-version-with-buildinfo/.no_build diff --git a/test/filesystem-version-with-buildinfo/fs-json-dubpackage-1.0.7+build-9-9-9.zip b/test/filesystem-version-with-buildinfo/fs-json-dubpackage-1.0.7+build-9-9-9.zip new file mode 100644 index 0000000..1b9915e --- /dev/null +++ b/test/filesystem-version-with-buildinfo/fs-json-dubpackage-1.0.7+build-9-9-9.zip Binary files differ diff --git a/test/injected-from-dependency/.no_test b/test/injected-from-dependency/.no_test new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/injected-from-dependency/.no_test diff --git a/test/injected-from-dependency/ahook.d b/test/injected-from-dependency/ahook.d new file mode 100644 index 0000000..bbc525c --- /dev/null +++ b/test/injected-from-dependency/ahook.d @@ -0,0 +1,6 @@ +module ahook; + +shared static this() { + import vars; + valueStoredHere = 1337; +} diff --git a/test/injected-from-dependency/dub.json b/test/injected-from-dependency/dub.json new file mode 100644 index 0000000..5c13700 --- /dev/null +++ b/test/injected-from-dependency/dub.json @@ -0,0 +1,18 @@ +{ + "description": "Test if source file expected to be injected was into binary", + "name": "injected-from-dependency", + "targetType": "executable", + + "dependencies": { + ":toload": "*" + }, + + "subPackages": [ + { + "name": "toload", + "sourcePaths": ["toload"], + "importPaths": ["toload"], + "injectSourceFiles": ["ahook.d"] + } + ] +} diff --git a/test/injected-from-dependency/source/entry.d b/test/injected-from-dependency/source/entry.d new file mode 100644 index 0000000..8a11113 --- /dev/null +++ b/test/injected-from-dependency/source/entry.d @@ -0,0 +1,4 @@ +void main() { + import vars; + assert(valueStoredHere == 1337); +} diff --git a/test/injected-from-dependency/toload/vars.d b/test/injected-from-dependency/toload/vars.d new file mode 100644 index 0000000..5c52075 --- /dev/null +++ b/test/injected-from-dependency/toload/vars.d @@ -0,0 +1,3 @@ +module vars; + +int valueStoredHere; diff --git a/test/issue2192-environment-variables.sh b/test/issue2192-environment-variables.sh new file mode 100755 index 0000000..df862ed --- /dev/null +++ b/test/issue2192-environment-variables.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +. $(dirname "${BASH_SOURCE[0]}")/common.sh + +if [ -n "${DUB_PACKAGE-}" ]; then + die $LINENO '$DUB_PACKAGE must not be set when running this test!' +fi + +if ! { $DUB build --force --root "$CURR_DIR/issue2192-environment-variables" --skip-registry=all; }; then + die $LINENO 'Failed to build package with built-in environment variables.' +fi + +if [ -s "$CURR_DIR/issue2192-environment-variables/package.txt" ]; then + rm "$CURR_DIR/issue2192-environment-variables/package.txt" +else + die $LINENO 'Expected generated package.txt file is missing.' +fi + +OUTPUT=$($DUB describe --root "$CURR_DIR/issue2192-environment-variables" --skip-registry=all --data=pre-build-commands --data-list) +if [ "$OUTPUT" != "echo 'issue2192-environment-variables' > package.txt" ]; then + die $LINENO 'describe did not contain subtituted values or the correct package name' +fi diff --git a/test/issue2192-environment-variables/.no_run b/test/issue2192-environment-variables/.no_run new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/issue2192-environment-variables/.no_run diff --git a/test/issue2192-environment-variables/dub.sdl b/test/issue2192-environment-variables/dub.sdl new file mode 100644 index 0000000..e885cb9 --- /dev/null +++ b/test/issue2192-environment-variables/dub.sdl @@ -0,0 +1,2 @@ +name "issue2192-environment-variables" +preBuildCommands "echo '$DUB_PACKAGE' > package.txt" diff --git a/test/issue2192-environment-variables/source/lib.d b/test/issue2192-environment-variables/source/lib.d new file mode 100644 index 0000000..1a804d9 --- /dev/null +++ b/test/issue2192-environment-variables/source/lib.d @@ -0,0 +1 @@ +module lib; diff --git a/test/run-unittest.d b/test/run-unittest.d index e2b2e26..71ee3f4 100644 --- a/test/run-unittest.d +++ b/test/run-unittest.d @@ -10,7 +10,7 @@ int main(string[] args) { - import std.algorithm : among; + import std.algorithm : among, endsWith; import std.file : dirEntries, DirEntry, exists, getcwd, readText, SpanMode; import std.format : format; import std.stdio : File, writeln; @@ -64,6 +64,8 @@ //** done foreach(DirEntry script; dirEntries(curr_dir, (args.length > 1) ? args[1] : "*.sh", SpanMode.shallow)) { + if (!script.name.endsWith(".sh")) + continue; if (baseName(script.name).among("run-unittest.sh", "common.sh")) continue; const min_frontend = script.name ~ ".min_frontend"; if (exists(min_frontend) && frontend.length && frontend < min_frontend.readText) continue; @@ -75,6 +77,8 @@ foreach (DirEntry script; dirEntries(curr_dir, (args.length > 1) ? args[1] : "*.script.d", SpanMode.shallow)) { + if (!script.name.endsWith(".d")) + continue; const min_frontend = script.name ~ ".min_frontend"; if (frontend.length && exists(min_frontend) && frontend < min_frontend.readText) continue; log("Running " ~ script ~ "..."); diff --git a/test/win32_default.d b/test/win32_default.d new file mode 100644 index 0000000..fcbfe5a --- /dev/null +++ b/test/win32_default.d @@ -0,0 +1,54 @@ +/+ dub.json: { + "name": "win32_default", + "configurations": [ + { + "name": "Default", + "versions": [ "Default" ] + }, + { + "name": "OMF", + "versions": [ "OMF" ] + }, + { + "name": "MsCoff", + "versions": [ "MsCoff" ] + }, + { + "name": "MsCoff64", + "versions": [ "MsCoff", "Is64" ] + } + ] +} +/ + +module dynlib.app; + +pragma(msg, "Frontend: ", __VERSION__); + +// Object format should match the expectation +version (OMF) +{ + enum expSize = 4; + enum expFormat = "omf"; +} +else version (MsCoff) +{ + // Should be a 32 bit build + version (Is64) enum expSize = 8; + else enum expSize = 4; + + enum expFormat = "coff"; +} +else version (Default) +{ + enum expSize = 4; + enum expFormat = __VERSION__ >= 2099 ? "coff" : "omf"; +} +else +{ + static assert(false, "Missing version flag!"); +} + +enum actFormat = __traits(getTargetInfo, "objectFormat"); + +static assert(actFormat == expFormat); +static assert((int*).sizeof == expSize); diff --git a/test/win32_default.script.d b/test/win32_default.script.d new file mode 100644 index 0000000..de0356e --- /dev/null +++ b/test/win32_default.script.d @@ -0,0 +1,89 @@ +/+ dub.json: { + "name": "win32_default_test" +} +/ + +module win32_default.script; + +int main() +{ + import std.stdio; + + version (Windows) + { + version (DigitalMars) + enum disabled = null; + else + enum disabled = "DMD as the host compiler"; + } + else + enum disabled = "Windows"; + + static if (disabled) + { + writeln("Test `win32_default` requires " ~ disabled); + return 0; + } + else + { + import std.algorithm; + import std.path; + import std.process; + + const dir = __FILE_FULL_PATH__.dirName(); + const file = buildPath(dir, "win32_default.d"); + + const dub = environment.get("DUB", buildPath(dirName(dir), "bin", "dub.exe")); + const dmd = environment.get("DMD", "dmd"); + + int exitCode; + + void runTest(scope const string[] cmd) + { + const result = execute(cmd); + + if (result.status || result.output.canFind("Failed")) + { + writefln("\n> %-(%s %)", cmd); + writeln("==========================================================="); + writeln(result.output); + writeln("==========================================================="); + writeln("Last command failed with exit code ", result.status, '\n'); + exitCode = 1; + } + } + + // Test without --arch + runTest([ + dub, "build", + "--compiler", dmd, + "--config", "MsCoff64", + "--single", file, + ]); + + // Test with different --arch + const string[2][] tests = [ + [ "x86", "Default" ], + [ "x86_omf", "OMF" ], + [ "x86_mscoff", "MsCoff" ], + [ "x86_64", "MsCoff64" ], + ]; + + foreach (string[2] test; tests) + { + const arch = test[0]; + const config = test[1]; + + runTest([ + dub, "build", + "--compiler", dmd, + "--arch", arch, + "--config", config, + "--single", file, + ]); + } + + + + return exitCode; + } +}