diff --git a/changelog/command-variables.dd b/changelog/command-variables.dd new file mode 100644 index 0000000..9d30128 --- /dev/null +++ b/changelog/command-variables.dd @@ -0,0 +1,20 @@ +Command environment variable substitution changed + +Now users can use the documented predefined variables inside custom command +directives without the need for a wrapper shell script. + +Before this would have failed: +```json +"preBuildCommands": ["$DC -run foo.d"] +``` +unless DC was defined as environment variable outside DUB. + +It was before possible to run a script that used the $DC environment variable or +on POSIX escape the `$` with `$$DC` to make the shell substitute the variable. +These workarounds are no longer needed now. + +API change: none of the different command directives are no longer substituted +with the process environment variables. You now access the raw commands as +provided by the user in the recipe. `dub describe` has been adjusted and now +also processes the predefined environment variables as well as the process +environment variables. diff --git a/source/dub/description.d b/source/dub/description.d index 2953107..0ebdd5b 100644 --- a/source/dub/description.d +++ b/source/dub/description.d @@ -90,12 +90,12 @@ string[] debugVersions; /// D debug version identifiers to set string[] importPaths; string[] stringImportPaths; - string[] preGenerateCommands; /// commands executed before creating the description - string[] postGenerateCommands; /// commands executed after creating the description - string[] preBuildCommands; /// Commands to execute prior to every build - string[] postBuildCommands; /// Commands to execute after every build - string[] preRunCommands; /// Commands to execute prior to every run - string[] postRunCommands; /// Commands to execute after every run + string[] preGenerateCommands; /// Commands executed before creating the description, with variables not substituted. + string[] postGenerateCommands; /// Commands executed after creating the description, with variables not substituted. + string[] preBuildCommands; /// Commands to execute prior to every build, with variables not substituted. + string[] postBuildCommands; /// Commands to execute after every build, with variables not substituted. + string[] preRunCommands; /// Commands to execute prior to every run, with variables not substituted. + string[] postRunCommands; /// Commands to execute after every run, with variables not substituted. string[string] environments; string[string] buildEnvironments; string[string] runEnvironments; diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 4251789..60be1b8 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -175,8 +175,7 @@ // run post-build commands if (!cached && buildsettings.postBuildCommands.length) { logInfo("Running post-build commands..."); - runBuildCommands(buildsettings.postBuildCommands, pack, m_project, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.postBuildEnvironments]); + runBuildCommands(CommandType.postBuild, buildsettings.postBuildCommands, pack, m_project, settings, buildsettings); } return cached; @@ -214,8 +213,7 @@ if( buildsettings.preBuildCommands.length ){ logInfo("Running pre-build commands..."); - runBuildCommands(buildsettings.preBuildCommands, pack, m_project, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preBuildEnvironments]); + runBuildCommands(CommandType.preBuild, buildsettings.preBuildCommands, pack, m_project, settings, buildsettings); } // override target path @@ -335,8 +333,7 @@ if( buildsettings.preBuildCommands.length ){ logInfo("Running pre-build commands..."); - runBuildCommands(buildsettings.preBuildCommands, pack, m_project, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preBuildEnvironments]); + runBuildCommands(CommandType.preBuild, buildsettings.preBuildCommands, pack, m_project, settings, buildsettings); } buildWithCompiler(settings, buildsettings); @@ -580,8 +577,7 @@ { if (buildsettings.preRunCommands.length) { logInfo("Running pre-run commands..."); - runBuildCommands(buildsettings.preRunCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.runEnvironments, buildsettings.preRunEnvironments]); + runBuildCommands(CommandType.preRun, buildsettings.preRunCommands, pack, proj, settings, buildsettings); } } @@ -590,8 +586,7 @@ { if (buildsettings.postRunCommands.length) { logInfo("Running post-run commands..."); - runBuildCommands(buildsettings.postRunCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.runEnvironments, buildsettings.postRunEnvironments]); + runBuildCommands(CommandType.postRun, buildsettings.postRunCommands, pack, proj, settings, buildsettings); } } diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index 2b4a7da..d471d63 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -832,8 +832,7 @@ { if (buildsettings.preGenerateCommands.length && !isRecursiveInvocation(pack.name)) { logInfo("Running pre-generate commands for %s...", pack.name); - runBuildCommands(buildsettings.preGenerateCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preGenerateEnvironments]); + runBuildCommands(CommandType.preGenerate, buildsettings.preGenerateCommands, pack, proj, settings, buildsettings); } } @@ -845,8 +844,7 @@ { if (buildsettings.postGenerateCommands.length && !isRecursiveInvocation(pack.name)) { logInfo("Running post-generate commands for %s...", pack.name); - runBuildCommands(buildsettings.postGenerateCommands, pack, proj, settings, buildsettings, - [buildsettings.environments, buildsettings.buildEnvironments, buildsettings.postGenerateEnvironments]); + runBuildCommands(CommandType.postGenerate, buildsettings.postGenerateCommands, pack, proj, settings, buildsettings); } if (generate_binary) { @@ -913,29 +911,44 @@ command execution loops. The latter could otherwise happen when a command runs "dub describe" or similar functionality. */ -void runBuildCommands(in string[] commands, in Package pack, in Project proj, +void runBuildCommands(CommandType type, in string[] commands, in Package pack, in Project proj, in GeneratorSettings settings, in BuildSettings build_settings, in string[string][] extraVars = null) { - import dub.internal.utils : getDUBExePath, runCommands; + import dub.internal.utils : runCommands; + + auto env = makeCommandEnvironmentVariables(type, pack, proj, settings, build_settings, extraVars); + auto sub_commands = processVars(proj, pack, settings, commands, false, env); + + auto depNames = proj.dependencies.map!((a) => a.name).array(); + storeRecursiveInvokations(env, proj.rootPackage.name ~ depNames); + + runCommands(sub_commands, env.collapseEnv, pack.path().toString()); +} + +const(string[string])[] makeCommandEnvironmentVariables(CommandType type, + in Package pack, in Project proj, in GeneratorSettings settings, + in BuildSettings build_settings, in string[string][] extraVars = null) +{ + import dub.internal.utils : getDUBExePath; import std.conv : to, text; import std.process : environment, escapeShellFileName; - string[string] env = environment.toAA(); + string[string] env; // TODO: do more elaborate things here // TODO: escape/quote individual items appropriately - env["VERSIONS"] = join(cast(string[])build_settings.versions," "); - env["LIBS"] = join(cast(string[])build_settings.libs," "); - env["SOURCE_FILES"] = join(cast(string[])build_settings.sourceFiles," "); - env["IMPORT_PATHS"] = join(cast(string[])build_settings.importPaths," "); - env["STRING_IMPORT_PATHS"] = join(cast(string[])build_settings.stringImportPaths," "); + env["VERSIONS"] = join(build_settings.versions, " "); + env["LIBS"] = join(build_settings.libs, " "); + env["SOURCE_FILES"] = join(build_settings.sourceFiles, " "); + env["IMPORT_PATHS"] = join(build_settings.importPaths, " "); + env["STRING_IMPORT_PATHS"] = join(build_settings.stringImportPaths, " "); env["DC"] = settings.platform.compilerBinary; env["DC_BASE"] = settings.platform.compiler; env["D_FRONTEND_VER"] = to!string(settings.platform.frontendVersion); env["DUB_EXE"] = getDUBExePath(settings.platform.compilerBinary); - env["DUB_PLATFORM"] = join(cast(string[])settings.platform.platform," "); - env["DUB_ARCH"] = join(cast(string[])settings.platform.architecture," "); + env["DUB_PLATFORM"] = join(settings.platform.platform, " "); + env["DUB_ARCH"] = join(settings.platform.architecture, " "); env["DUB_TARGET_TYPE"] = to!string(build_settings.targetType); env["DUB_TARGET_PATH"] = build_settings.targetPath; @@ -968,15 +981,49 @@ env["DUB_ROOT_PACKAGE_TARGET_TYPE"] = to!string(rootPackageBuildSettings.targetType); env["DUB_ROOT_PACKAGE_TARGET_PATH"] = rootPackageBuildSettings.targetPath; env["DUB_ROOT_PACKAGE_TARGET_NAME"] = rootPackageBuildSettings.targetName; - - foreach (aa; extraVars) { - foreach (k, v; aa) - env[k] = v; + + const(string[string])[] typeEnvVars; + with (build_settings) final switch (type) + { + // pre/postGenerate don't have generateEnvironments, but reuse buildEnvironments + case CommandType.preGenerate: typeEnvVars = [environments, buildEnvironments, preGenerateEnvironments]; break; + case CommandType.postGenerate: typeEnvVars = [environments, buildEnvironments, postGenerateEnvironments]; break; + case CommandType.preBuild: typeEnvVars = [environments, buildEnvironments, preBuildEnvironments]; break; + case CommandType.postBuild: typeEnvVars = [environments, buildEnvironments, postBuildEnvironments]; break; + case CommandType.preRun: typeEnvVars = [environments, runEnvironments, preRunEnvironments]; break; + case CommandType.postRun: typeEnvVars = [environments, runEnvironments, postRunEnvironments]; break; } - auto depNames = proj.dependencies.map!((a) => a.name).array(); - storeRecursiveInvokations(env, proj.rootPackage.name ~ depNames); - runCommands(commands, env, pack.path().toString()); + return [environment.toAA()] ~ env ~ typeEnvVars ~ extraVars; +} + +string[string] collapseEnv(in string[string][] envs) +{ + string[string] ret; + foreach (subEnv; envs) + { + foreach (k, v; subEnv) + ret[k] = v; + } + return ret; +} + +/// Type to specify where CLI commands that need to be run came from. Needed for +/// proper substitution with support for the different environments. +enum CommandType +{ + /// Defined in the preGenerateCommands setting + preGenerate, + /// Defined in the postGenerateCommands setting + postGenerate, + /// Defined in the preBuildCommands setting + preBuild, + /// Defined in the postBuildCommands setting + postBuild, + /// Defined in the preRunCommands setting + preRun, + /// Defined in the postRunCommands setting + postRun } private bool isRecursiveInvocation(string pack) @@ -985,20 +1032,22 @@ import std.process : environment; return environment - .get("DUB_PACKAGES_USED", "") - .splitter(",") - .canFind(pack); + .get("DUB_PACKAGES_USED", "") + .splitter(",") + .canFind(pack); } -private void storeRecursiveInvokations(string[string] env, string[] packs) +private void storeRecursiveInvokations(ref const(string[string])[] env, string[] packs) { import std.algorithm : canFind, splitter; import std.range : chain; import std.process : environment; - env["DUB_PACKAGES_USED"] = environment - .get("DUB_PACKAGES_USED", "") - .splitter(",") - .chain(packs) - .join(","); + env ~= [ + "DUB_PACKAGES_USED": environment + .get("DUB_PACKAGES_USED", "") + .splitter(",") + .chain(packs) + .join(",") + ]; } diff --git a/source/dub/project.d b/source/dub/project.d index 026697d..0ef3fea 100644 --- a/source/dub/project.d +++ b/source/dub/project.d @@ -1206,13 +1206,6 @@ dst.addPostRunEnvironments(processVerEnvs(settings.postRunEnvironments, gsettings.buildSettings.postRunEnvironments)); auto buildEnvs = [dst.environments, dst.buildEnvironments]; - auto runEnvs = [dst.environments, dst.runEnvironments]; - auto preGenEnvs = [dst.environments, dst.preGenerateEnvironments]; - auto postGenEnvs = [dst.environments, dst.postGenerateEnvironments]; - auto preBuildEnvs = buildEnvs ~ [dst.preBuildEnvironments]; - auto postBuildEnvs = buildEnvs ~ [dst.postBuildEnvironments]; - auto preRunEnvs = runEnvs ~ [dst.preRunEnvironments]; - auto postRunEnvs = runEnvs ~ [dst.postRunEnvironments]; dst.addDFlags(processVars(project, pack, gsettings, settings.dflags, false, buildEnvs)); dst.addLFlags(processVars(project, pack, gsettings, settings.lflags, false, buildEnvs)); @@ -1228,15 +1221,17 @@ dst.addDebugVersionFilters(processVars(project, pack, gsettings, settings.debugVersionFilters, false, buildEnvs)); dst.addImportPaths(processVars(project, pack, gsettings, settings.importPaths, true, buildEnvs)); dst.addStringImportPaths(processVars(project, pack, gsettings, settings.stringImportPaths, true, buildEnvs)); - dst.addPreGenerateCommands(processVars(project, pack, gsettings, settings.preGenerateCommands, false, preGenEnvs)); - dst.addPostGenerateCommands(processVars(project, pack, gsettings, settings.postGenerateCommands, false, postGenEnvs)); - dst.addPreBuildCommands(processVars(project, pack, gsettings, settings.preBuildCommands, false, preBuildEnvs)); - dst.addPostBuildCommands(processVars(project, pack, gsettings, settings.postBuildCommands, false, postBuildEnvs)); - dst.addPreRunCommands(processVars(project, pack, gsettings, settings.preRunCommands, false, preRunEnvs)); - dst.addPostRunCommands(processVars(project, pack, gsettings, settings.postRunCommands, false, postRunEnvs)); dst.addRequirements(settings.requirements); dst.addOptions(settings.options); + // commands are substituted in dub.generators.generator : runBuildCommands + dst.addPreGenerateCommands(settings.preGenerateCommands); + dst.addPostGenerateCommands(settings.postGenerateCommands); + dst.addPreBuildCommands(settings.preBuildCommands); + dst.addPostBuildCommands(settings.postBuildCommands); + dst.addPreRunCommands(settings.preRunCommands); + dst.addPostRunCommands(settings.postRunCommands); + if (include_target_settings) { dst.targetType = settings.targetType; dst.targetPath = processVars(settings.targetPath, project, pack, gsettings, true, buildEnvs); @@ -1248,13 +1243,13 @@ } } -private string[] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, string[] vars, bool are_paths = false, in string[string][] extraVers = null) +string[] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, in string[] vars, bool are_paths = false, in string[string][] extraVers = null) { auto ret = appender!(string[])(); processVars!glob(ret, project, pack, gsettings, vars, are_paths, extraVers); return ret.data; } -private void processVars(bool glob = false)(ref Appender!(string[]) dst, in Project project, in Package pack, in GeneratorSettings gsettings, string[] vars, bool are_paths = false, in string[string][] extraVers = null) +void processVars(bool glob = false)(ref Appender!(string[]) dst, in Project project, in Package pack, in GeneratorSettings gsettings, in string[] vars, bool are_paths = false, in string[string][] extraVers = null) { static if (glob) alias process = processVarsWithGlob!(Project, Package); @@ -1264,7 +1259,7 @@ dst.put(process(var, project, pack, gsettings, are_paths, extraVers)); } -private string processVars(Project, Package)(string var, in Project project, in Package pack, in GeneratorSettings gsettings, bool is_path, in string[string][] extraVers = null) +string processVars(Project, Package)(string var, in Project project, in Package pack, in GeneratorSettings gsettings, bool is_path, in string[string][] extraVers = null) { var = var.expandVars!(varName => getVariable(varName, project, pack, gsettings, extraVers)); if (!is_path) @@ -1275,13 +1270,13 @@ else return p.toNativeString(); } -private string[string] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, string[string] vars, in string[string][] extraVers = null) +string[string] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, in string[string] vars, in string[string][] extraVers = null) { string[string] ret; processVars!glob(ret, project, pack, gsettings, vars, extraVers); return ret; } -private void processVars(bool glob = false)(ref string[string] dst, in Project project, in Package pack, in GeneratorSettings gsettings, string[string] vars, in string[string][] extraVers) +void processVars(bool glob = false)(ref string[string] dst, in Project project, in Package pack, in GeneratorSettings gsettings, in string[string] vars, in string[string][] extraVers) { static if (glob) alias process = processVarsWithGlob!(Project, Package); diff --git a/test/issue2192-environment-variables.sh b/test/issue2192-environment-variables.sh new file mode 100755 index 0000000..5d36a47 --- /dev/null +++ b/test/issue2192-environment-variables.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +. $(dirname "${BASH_SOURCE[0]}")/common.sh + +if [ -n "${DUB_PACKAGE-}" ]; then + die $LINENO '$DUB_PACKAGE must not be set when running this test!' +fi + +if ! { $DUB build --force --root "$CURR_DIR/issue2192-environment-variables" --skip-registry=all; }; then + die $LINENO 'Failed to build package with built-in environment variables.' +fi + +if [ -s "$CURR_DIR/issue2192-environment-variables/package.txt" ]; then + rm "$CURR_DIR/issue2192-environment-variables/package.txt" +else + die $LINENO 'Expected generated package.txt file is missing.' +fi diff --git a/test/issue2192-environment-variables/.no_run b/test/issue2192-environment-variables/.no_run new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/issue2192-environment-variables/.no_run diff --git a/test/issue2192-environment-variables/dub.sdl b/test/issue2192-environment-variables/dub.sdl new file mode 100644 index 0000000..e885cb9 --- /dev/null +++ b/test/issue2192-environment-variables/dub.sdl @@ -0,0 +1,2 @@ +name "issue2192-environment-variables" +preBuildCommands "echo '$DUB_PACKAGE' > package.txt" diff --git a/test/issue2192-environment-variables/source/lib.d b/test/issue2192-environment-variables/source/lib.d new file mode 100644 index 0000000..1a804d9 --- /dev/null +++ b/test/issue2192-environment-variables/source/lib.d @@ -0,0 +1 @@ +module lib;