Fork: 0
johnpearcey / dub_jkp
Browse code
move command variable substitution to execution
This is needed as commands that need to be run need to have generator
settings available before running to avoid inconsistencies with
environment variables passed to the actually ran program and variables
used in substitution.

Fixes #2192

Fixes pre/postGenerateCommands inconsistently using dub environments
(execution was substituted including buildEnvironments, but shell cmd
did not get the build variables for substitution) - now both get
the build variables. Solved by removing code duplication. (env building)

This commit changes the behavior of `dub describe` so that describing
commands settings no longer substitutes variables. Before, known
variables would have been replaced in the output and on unknown
variables `dub describe` would fail with an error
`Target 'x' doesn't exist`

Now exposes some previously private project.d processVars functions
master
1 parent 7ee2cbe commit 6c21e598a61a5f5561409d526591101f111d0d3d
@WebFreak001 WebFreak001 authored on 13 Feb 2022
Jan Jurzitza committed on 14 Feb 2022
Patch
Unified Split
Showing 9 changed files
View
21
■■■■■
 changelog/command-variables.dd 0 → 100644
Command environment variable substitution changed
 
Now users can use the documented predefined variables inside custom command
directives without the need for a wrapper shell script.
 
Before this would have failed:
```json
"preBuildCommands": ["$DC -run foo.d"]
```
unless DC was defined as environment variable outside DUB.
 
It was before possible to run a script that used the $DC environment variable or
on POSIX escape the `$` with `$$DC` to make the shell substitute the variable.
These workarounds are no longer needed now.
 
API change: none of the different command directives are no longer substituted
with the process environment variables. You now access the raw commands as
provided by the user in the recipe. `dub describe` has been adjusted and now
also processes the predefined environment variables as well as the process
environment variables.
View
12
■■■■■
 source/dub/description.d
string[] versions; /// D version identifiers to set
string[] debugVersions; /// D debug version identifiers to set
string[] importPaths;
string[] stringImportPaths;
string[] preGenerateCommands; /// commands executed before creating the description
string[] postGenerateCommands; /// commands executed after creating the description
string[] preBuildCommands; /// Commands to execute prior to every build
string[] postBuildCommands; /// Commands to execute after every build
string[] preRunCommands; /// Commands to execute prior to every run
string[] postRunCommands; /// Commands to execute after every run
string[] preGenerateCommands; /// Commands executed before creating the description, with variables not substituted.
string[] postGenerateCommands; /// Commands executed after creating the description, with variables not substituted.
string[] preBuildCommands; /// Commands to execute prior to every build, with variables not substituted.
string[] postBuildCommands; /// Commands to execute after every build, with variables not substituted.
string[] preRunCommands; /// Commands to execute prior to every run, with variables not substituted.
string[] postRunCommands; /// Commands to execute after every run, with variables not substituted.
string[string] environments;
string[string] buildEnvironments;
string[string] runEnvironments;
string[string] preGenerateEnvironments;
View
20
■■■■■
 source/dub/generators/build.d
 
// run post-build commands
if (!cached && buildsettings.postBuildCommands.length) {
logInfo("Running post-build commands...");
runBuildCommands(buildsettings.postBuildCommands, pack, m_project, settings, buildsettings,
[buildsettings.environments, buildsettings.buildEnvironments, buildsettings.postBuildEnvironments]);
runBuildCommands(CommandType.postBuild, buildsettings.postBuildCommands, pack, m_project, settings, buildsettings);
}
 
return cached;
}
logInfo("%s %s: building configuration \"%s\"...", pack.name, pack.version_, config);
 
if( buildsettings.preBuildCommands.length ){
logInfo("Running pre-build commands...");
runBuildCommands(buildsettings.preBuildCommands, pack, m_project, settings, buildsettings,
[buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preBuildEnvironments]);
runBuildCommands(CommandType.preBuild, buildsettings.preBuildCommands, pack, m_project, settings, buildsettings);
}
 
// override target path
auto cbuildsettings = buildsettings;
}
 
if( buildsettings.preBuildCommands.length ){
logInfo("Running pre-build commands...");
runBuildCommands(buildsettings.preBuildCommands, pack, m_project, settings, buildsettings,
[buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preBuildEnvironments]);
runBuildCommands(CommandType.preBuild, buildsettings.preBuildCommands, pack, m_project, settings, buildsettings);
}
 
buildWithCompiler(settings, buildsettings);
 
in BuildSettings buildsettings)
{
if (buildsettings.preRunCommands.length) {
logInfo("Running pre-run commands...");
runBuildCommands(buildsettings.preRunCommands, pack, proj, settings, buildsettings,
[buildsettings.environments, buildsettings.runEnvironments, buildsettings.preRunEnvironments]);
runBuildCommands(CommandType.preRun, buildsettings.preRunCommands, pack, proj, settings, buildsettings);
}
}
 
private void runPostRunCommands(in Package pack, in Project proj, in GeneratorSettings settings,
in BuildSettings buildsettings)
{
if (buildsettings.postRunCommands.length) {
logInfo("Running post-run commands...");
runBuildCommands(buildsettings.postRunCommands, pack, proj, settings, buildsettings,
[buildsettings.environments, buildsettings.runEnvironments, buildsettings.postRunEnvironments]);
runBuildCommands(CommandType.postRun, buildsettings.postRunCommands, pack, proj, settings, buildsettings);
}
}
 
private void cleanupTemporaries()
View
174
■■■■■
 source/dub/generators/generator.d
in BuildSettings buildsettings)
{
if (buildsettings.preGenerateCommands.length && !isRecursiveInvocation(pack.name)) {
logInfo("Running pre-generate commands for %s...", pack.name);
runBuildCommands(buildsettings.preGenerateCommands, pack, proj, settings, buildsettings,
[buildsettings.environments, buildsettings.buildEnvironments, buildsettings.preGenerateEnvironments]);
runBuildCommands(CommandType.preGenerate, buildsettings.preGenerateCommands, pack, proj, settings, buildsettings);
}
}
 
/**
in BuildSettings buildsettings, NativePath target_path, bool generate_binary)
{
if (buildsettings.postGenerateCommands.length && !isRecursiveInvocation(pack.name)) {
logInfo("Running post-generate commands for %s...", pack.name);
runBuildCommands(buildsettings.postGenerateCommands, pack, proj, settings, buildsettings,
[buildsettings.environments, buildsettings.buildEnvironments, buildsettings.postGenerateEnvironments]);
runBuildCommands(CommandType.postGenerate, buildsettings.postGenerateCommands, pack, proj, settings, buildsettings);
}
 
if (generate_binary) {
if (!exists(buildsettings.targetPath))
that recursive dub invocations are detected and don't result in infinite
command execution loops. The latter could otherwise happen when a command
runs "dub describe" or similar functionality.
*/
void runBuildCommands(in string[] commands, in Package pack, in Project proj,
void runBuildCommands(CommandType type, in string[] commands, in Package pack, in Project proj,
in GeneratorSettings settings, in BuildSettings build_settings, in string[string][] extraVars = null)
{
import dub.internal.utils : getDUBExePath, runCommands;
import dub.internal.utils : runCommands;
 
auto env = makeCommandEnvironmentVariables(type, pack, proj, settings, build_settings, extraVars);
auto sub_commands = processVars(proj, pack, settings, commands, false, env);
 
auto depNames = proj.dependencies.map!((a) => a.name).array();
storeRecursiveInvokations(env, proj.rootPackage.name ~ depNames);
 
runCommands(sub_commands, env.collapseEnv, pack.path().toString());
}
 
const(string[string])[] makeCommandEnvironmentVariables(CommandType type,
in Package pack, in Project proj, in GeneratorSettings settings,
in BuildSettings build_settings, in string[string][] extraVars = null)
{
import dub.internal.utils : getDUBExePath;
import std.conv : to, text;
import std.process : environment, escapeShellFileName;
 
string[string] env = environment.toAA();
string[string] env;
// TODO: do more elaborate things here
// TODO: escape/quote individual items appropriately
env["VERSIONS"] = join(cast(string[])build_settings.versions," ");
env["LIBS"] = join(cast(string[])build_settings.libs," ");
env["SOURCE_FILES"] = join(cast(string[])build_settings.sourceFiles," ");
env["IMPORT_PATHS"] = join(cast(string[])build_settings.importPaths," ");
env["STRING_IMPORT_PATHS"] = join(cast(string[])build_settings.stringImportPaths," ");
env["VERSIONS"] = join(build_settings.versions, " ");
env["LIBS"] = join(build_settings.libs, " ");
env["SOURCE_FILES"] = join(build_settings.sourceFiles, " ");
env["IMPORT_PATHS"] = join(build_settings.importPaths, " ");
env["STRING_IMPORT_PATHS"] = join(build_settings.stringImportPaths, " ");
 
env["DC"] = settings.platform.compilerBinary;
env["DC_BASE"] = settings.platform.compiler;
env["D_FRONTEND_VER"] = to!string(settings.platform.frontendVersion);
 
env["DUB_EXE"] = getDUBExePath(settings.platform.compilerBinary);
env["DUB_PLATFORM"] = join(cast(string[])settings.platform.platform," ");
env["DUB_ARCH"] = join(cast(string[])settings.platform.architecture," ");
env["DUB_PLATFORM"] = join(settings.platform.platform, " ");
env["DUB_ARCH"] = join(settings.platform.architecture, " ");
 
env["DUB_TARGET_TYPE"] = to!string(build_settings.targetType);
env["DUB_TARGET_PATH"] = build_settings.targetPath;
env["DUB_TARGET_NAME"] = build_settings.targetName;
auto rootPackageBuildSettings = proj.rootPackage.getBuildSettings(settings.platform, cfgs[proj.rootPackage.name]);
env["DUB_ROOT_PACKAGE_TARGET_TYPE"] = to!string(rootPackageBuildSettings.targetType);
env["DUB_ROOT_PACKAGE_TARGET_PATH"] = rootPackageBuildSettings.targetPath;
env["DUB_ROOT_PACKAGE_TARGET_NAME"] = rootPackageBuildSettings.targetName;
foreach (aa; extraVars) {
foreach (k, v; aa)
env[k] = v;
}
 
auto depNames = proj.dependencies.map!((a) => a.name).array();
storeRecursiveInvokations(env, proj.rootPackage.name ~ depNames);
runCommands(commands, env, pack.path().toString());
 
const(string[string])[] typeEnvVars;
with (build_settings) final switch (type)
{
// pre/postGenerate don't have generateEnvironments, but reuse buildEnvironments
case CommandType.preGenerate: typeEnvVars = [environments, buildEnvironments, preGenerateEnvironments]; break;
case CommandType.postGenerate: typeEnvVars = [environments, buildEnvironments, postGenerateEnvironments]; break;
case CommandType.preBuild: typeEnvVars = [environments, buildEnvironments, preBuildEnvironments]; break;
case CommandType.postBuild: typeEnvVars = [environments, buildEnvironments, postBuildEnvironments]; break;
case CommandType.preRun: typeEnvVars = [environments, runEnvironments, preRunEnvironments]; break;
case CommandType.postRun: typeEnvVars = [environments, runEnvironments, postRunEnvironments]; break;
}
 
return [environment.toAA()] ~ env ~ typeEnvVars ~ extraVars;
}
 
string[string] collapseEnv(in string[string][] envs)
{
string[string] ret;
foreach (subEnv; envs)
{
foreach (k, v; subEnv)
ret[k] = v;
}
return ret;
}
 
/// Type to specify where CLI commands that need to be run came from. Needed for
/// proper substitution with support for the different environments.
enum CommandType
{
/// Defined in the preGenerateCommands setting
preGenerate,
/// Defined in the postGenerateCommands setting
postGenerate,
/// Defined in the preBuildCommands setting
preBuild,
/// Defined in the postBuildCommands setting
postBuild,
/// Defined in the preRunCommands setting
preRun,
/// Defined in the postRunCommands setting
postRun
}
 
private bool isRecursiveInvocation(string pack)
{
import std.algorithm : canFind, splitter;
import std.process : environment;
 
return environment
.get("DUB_PACKAGES_USED", "")
.splitter(",")
.canFind(pack);
}
 
private void storeRecursiveInvokations(string[string] env, string[] packs)
.get("DUB_PACKAGES_USED", "")
.splitter(",")
.canFind(pack);
}
 
private void storeRecursiveInvokations(ref const(string[string])[] env, string[] packs)
{
import std.algorithm : canFind, splitter;
import std.range : chain;
import std.process : environment;
 
env["DUB_PACKAGES_USED"] = environment
.get("DUB_PACKAGES_USED", "")
.splitter(",")
.chain(packs)
.join(",");
}
env ~= [
"DUB_PACKAGES_USED": environment
.get("DUB_PACKAGES_USED", "")
.splitter(",")
.chain(packs)
.join(",")
];
}
View
31
■■■■■
 source/dub/project.d
dst.addPreRunEnvironments(processVerEnvs(settings.preRunEnvironments, gsettings.buildSettings.preRunEnvironments));
dst.addPostRunEnvironments(processVerEnvs(settings.postRunEnvironments, gsettings.buildSettings.postRunEnvironments));
 
auto buildEnvs = [dst.environments, dst.buildEnvironments];
auto runEnvs = [dst.environments, dst.runEnvironments];
auto preGenEnvs = [dst.environments, dst.preGenerateEnvironments];
auto postGenEnvs = [dst.environments, dst.postGenerateEnvironments];
auto preBuildEnvs = buildEnvs ~ [dst.preBuildEnvironments];
auto postBuildEnvs = buildEnvs ~ [dst.postBuildEnvironments];
auto preRunEnvs = runEnvs ~ [dst.preRunEnvironments];
auto postRunEnvs = runEnvs ~ [dst.postRunEnvironments];
 
dst.addDFlags(processVars(project, pack, gsettings, settings.dflags, false, buildEnvs));
dst.addLFlags(processVars(project, pack, gsettings, settings.lflags, false, buildEnvs));
dst.addLibs(processVars(project, pack, gsettings, settings.libs, false, buildEnvs));
dst.addVersionFilters(processVars(project, pack, gsettings, settings.versionFilters, false, buildEnvs));
dst.addDebugVersionFilters(processVars(project, pack, gsettings, settings.debugVersionFilters, false, buildEnvs));
dst.addImportPaths(processVars(project, pack, gsettings, settings.importPaths, true, buildEnvs));
dst.addStringImportPaths(processVars(project, pack, gsettings, settings.stringImportPaths, true, buildEnvs));
dst.addPreGenerateCommands(processVars(project, pack, gsettings, settings.preGenerateCommands, false, preGenEnvs));
dst.addPostGenerateCommands(processVars(project, pack, gsettings, settings.postGenerateCommands, false, postGenEnvs));
dst.addPreBuildCommands(processVars(project, pack, gsettings, settings.preBuildCommands, false, preBuildEnvs));
dst.addPostBuildCommands(processVars(project, pack, gsettings, settings.postBuildCommands, false, postBuildEnvs));
dst.addPreRunCommands(processVars(project, pack, gsettings, settings.preRunCommands, false, preRunEnvs));
dst.addPostRunCommands(processVars(project, pack, gsettings, settings.postRunCommands, false, postRunEnvs));
dst.addRequirements(settings.requirements);
dst.addOptions(settings.options);
 
// commands are substituted in dub.generators.generator : runBuildCommands
dst.addPreGenerateCommands(settings.preGenerateCommands);
dst.addPostGenerateCommands(settings.postGenerateCommands);
dst.addPreBuildCommands(settings.preBuildCommands);
dst.addPostBuildCommands(settings.postBuildCommands);
dst.addPreRunCommands(settings.preRunCommands);
dst.addPostRunCommands(settings.postRunCommands);
 
if (include_target_settings) {
dst.targetType = settings.targetType;
dst.targetPath = processVars(settings.targetPath, project, pack, gsettings, true, buildEnvs);
dst.mainSourceFile = processVars(settings.mainSourceFile, project, pack, gsettings, true, buildEnvs);
}
}
 
private string[] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, string[] vars, bool are_paths = false, in string[string][] extraVers = null)
string[] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, in string[] vars, bool are_paths = false, in string[string][] extraVers = null)
{
auto ret = appender!(string[])();
processVars!glob(ret, project, pack, gsettings, vars, are_paths, extraVers);
return ret.data;
}
private void processVars(bool glob = false)(ref Appender!(string[]) dst, in Project project, in Package pack, in GeneratorSettings gsettings, string[] vars, bool are_paths = false, in string[string][] extraVers = null)
void processVars(bool glob = false)(ref Appender!(string[]) dst, in Project project, in Package pack, in GeneratorSettings gsettings, in string[] vars, bool are_paths = false, in string[string][] extraVers = null)
{
static if (glob)
alias process = processVarsWithGlob!(Project, Package);
else
foreach (var; vars)
dst.put(process(var, project, pack, gsettings, are_paths, extraVers));
}
 
private string processVars(Project, Package)(string var, in Project project, in Package pack, in GeneratorSettings gsettings, bool is_path, in string[string][] extraVers = null)
string processVars(Project, Package)(string var, in Project project, in Package pack, in GeneratorSettings gsettings, bool is_path, in string[string][] extraVers = null)
{
var = var.expandVars!(varName => getVariable(varName, project, pack, gsettings, extraVers));
if (!is_path)
return var;
return (pack.path ~ p).toNativeString();
else
return p.toNativeString();
}
private string[string] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, string[string] vars, in string[string][] extraVers = null)
string[string] processVars(bool glob = false)(in Project project, in Package pack, in GeneratorSettings gsettings, in string[string] vars, in string[string][] extraVers = null)
{
string[string] ret;
processVars!glob(ret, project, pack, gsettings, vars, extraVers);
return ret;
}
private void processVars(bool glob = false)(ref string[string] dst, in Project project, in Package pack, in GeneratorSettings gsettings, string[string] vars, in string[string][] extraVers)
void processVars(bool glob = false)(ref string[string] dst, in Project project, in Package pack, in GeneratorSettings gsettings, in string[string] vars, in string[string][] extraVers)
{
static if (glob)
alias process = processVarsWithGlob!(Project, Package);
else
View
17
■■■■■
 test/issue2192-environment-variables.sh 0 → 100755
#!/usr/bin/env bash
. $(dirname "${BASH_SOURCE[0]}")/common.sh
 
if [ -n "${DUB_PACKAGE-}" ]; then
die $LINENO '$DUB_PACKAGE must not be set when running this test!'
fi
 
if ! { $DUB build --force --root "$CURR_DIR/issue2192-environment-variables" --skip-registry=all; }; then
die $LINENO 'Failed to build package with built-in environment variables.'
fi
 
if [ -s "$CURR_DIR/issue2192-environment-variables/package.txt" ]; then
rm "$CURR_DIR/issue2192-environment-variables/package.txt"
else
die $LINENO 'Expected generated package.txt file is missing.'
fi
View
test/issue2192-environment-variables/.no_run 0 → 100644
View
test/issue2192-environment-variables/dub.sdl 0 → 100644
View
test/issue2192-environment-variables/source/lib.d 0 → 100644