Newer
Older
dub_jkp / source / dub / generators / build.d
  1. /**
  2. Generator for direct compiler builds.
  3. Copyright: © 2013-2013 rejectedsoftware e.K.
  4. License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
  5. Authors: Sönke Ludwig
  6. */
  7. module dub.generators.build;
  8.  
  9. import dub.compilers.compiler;
  10. import dub.generators.generator;
  11. import dub.internal.utils;
  12. import dub.internal.vibecompat.core.file;
  13. import dub.internal.vibecompat.core.log;
  14. import dub.internal.vibecompat.inet.path;
  15. import dub.package_;
  16. import dub.packagemanager;
  17. import dub.project;
  18.  
  19. import std.algorithm;
  20. import std.array;
  21. import std.conv;
  22. import std.exception;
  23. import std.file;
  24. import std.process;
  25. import std.string;
  26. import std.encoding : sanitize;
  27.  
  28. class BuildGenerator : ProjectGenerator {
  29. private {
  30. Project m_project;
  31. PackageManager m_packageMan;
  32. Path[] m_temporaryFiles;
  33. }
  34.  
  35. this(Project app, PackageManager mgr)
  36. {
  37. super(app);
  38. m_project = app;
  39. m_packageMan = mgr;
  40. }
  41.  
  42. override void generateTargets(GeneratorSettings settings, in TargetInfo[string] targets)
  43. {
  44. scope (exit) cleanupTemporaries();
  45.  
  46. bool[string] visited;
  47. void buildTargetRec(string target)
  48. {
  49. if (target in visited) return;
  50. visited[target] = true;
  51.  
  52. auto ti = targets[target];
  53.  
  54. foreach (dep; ti.dependencies)
  55. buildTargetRec(dep);
  56.  
  57. auto bs = ti.buildSettings.dup;
  58. if (bs.targetType != TargetType.staticLibrary)
  59. foreach (ldep; ti.linkDependencies) {
  60. auto dbs = targets[ldep].buildSettings;
  61. bs.addSourceFiles((Path(dbs.targetPath) ~ getTargetFileName(dbs, settings.platform)).toNativeString());
  62. }
  63. buildTarget(settings, bs, ti.pack, ti.config);
  64. }
  65.  
  66. // build all targets
  67. if (settings.rdmd) {
  68. // RDMD always builds everything at once
  69. auto ti = targets[m_project.rootPackage.name];
  70. buildTarget(settings, ti.buildSettings.dup, m_project.rootPackage, ti.config);
  71. } else buildTargetRec(m_project.rootPackage.name);
  72.  
  73. // run the generated executable
  74. auto buildsettings = targets[m_project.rootPackage.name].buildSettings;
  75. if (settings.run && !(buildsettings.options & BuildOptions.syntaxOnly)) {
  76. auto exe_file_path = Path(buildsettings.targetPath) ~ getTargetFileName(buildsettings, settings.platform);
  77. runTarget(exe_file_path, buildsettings, settings.runArgs, settings);
  78. }
  79. }
  80.  
  81. private void buildTarget(GeneratorSettings settings, BuildSettings buildsettings, in Package pack, string config)
  82. {
  83. auto cwd = Path(getcwd());
  84. bool generate_binary = !(buildsettings.options & BuildOptions.syntaxOnly);
  85.  
  86. auto build_id = computeBuildID(config, buildsettings, settings);
  87.  
  88. // make all paths relative to shrink the command line
  89. string makeRelative(string path) { auto p = Path(path); if (p.absolute) p = p.relativeTo(cwd); return p.toNativeString(); }
  90. foreach (ref f; buildsettings.sourceFiles) f = makeRelative(f);
  91. foreach (ref p; buildsettings.importPaths) p = makeRelative(p);
  92. foreach (ref p; buildsettings.stringImportPaths) p = makeRelative(p);
  93.  
  94. // perform the actual build
  95. if (settings.rdmd) performRDMDBuild(settings, buildsettings, pack, config);
  96. else if (settings.direct || !generate_binary) performDirectBuild(settings, buildsettings, pack, config);
  97. else performCachedBuild(settings, buildsettings, pack, config, build_id);
  98.  
  99. // run post-build commands
  100. if (buildsettings.postBuildCommands.length) {
  101. logInfo("Running post-build commands...");
  102. runBuildCommands(buildsettings.postBuildCommands, buildsettings);
  103. }
  104. }
  105.  
  106. void performCachedBuild(GeneratorSettings settings, BuildSettings buildsettings, in Package pack, string config, string build_id)
  107. {
  108. auto cwd = Path(getcwd());
  109. auto target_path = pack.path ~ format(".dub/build/%s/", build_id);
  110.  
  111. if (!settings.force && isUpToDate(target_path, buildsettings, settings.platform)) {
  112. logInfo("Target %s (%s) is up to date. Use --force to rebuild.", pack.name, pack.vers);
  113. logDiagnostic("Using existing build in %s.", target_path.toNativeString());
  114. copyTargetFile(target_path, buildsettings, settings.platform);
  115. return;
  116. }
  117.  
  118. if (!isWritableDir(target_path, true)) {
  119. logInfo("Build directory %s is not writable. Falling back to direct build in the system's temp folder.", target_path.relativeTo(cwd).toNativeString());
  120. performDirectBuild(settings, buildsettings, pack, config);
  121. return;
  122. }
  123.  
  124. // determine basic build properties
  125. auto generate_binary = !(buildsettings.options & BuildOptions.syntaxOnly);
  126.  
  127. logInfo("Building %s configuration \"%s\", build type %s.", pack.name, config, settings.buildType);
  128.  
  129. if( buildsettings.preBuildCommands.length ){
  130. logInfo("Running pre-build commands...");
  131. runBuildCommands(buildsettings.preBuildCommands, buildsettings);
  132. }
  133.  
  134. // override target path
  135. auto cbuildsettings = buildsettings;
  136. cbuildsettings.targetPath = target_path.relativeTo(cwd).toNativeString();
  137. buildWithCompiler(settings, cbuildsettings);
  138.  
  139. copyTargetFile(target_path, buildsettings, settings.platform);
  140. }
  141.  
  142. void performRDMDBuild(GeneratorSettings settings, ref BuildSettings buildsettings, in Package pack, string config)
  143. {
  144. auto cwd = Path(getcwd());
  145. //Added check for existance of [AppNameInPackagejson].d
  146. //If exists, use that as the starting file.
  147. Path mainsrc;
  148. if (buildsettings.mainSourceFile.length) {
  149. mainsrc = Path(buildsettings.mainSourceFile);
  150. if (!mainsrc.absolute) mainsrc = pack.path ~ mainsrc;
  151. } else {
  152. logWarn(`Package has no "mainSourceFile" defined. Trying to guess something to pass to RDMD...`);
  153. mainsrc = getMainSourceFile(pack);
  154. }
  155.  
  156. // do not pass all source files to RDMD, only the main source file
  157. buildsettings.sourceFiles = buildsettings.sourceFiles.filter!(s => !s.endsWith(".d"))().array();
  158. settings.compiler.prepareBuildSettings(buildsettings, BuildSetting.commandLine);
  159.  
  160. auto generate_binary = !buildsettings.dflags.canFind("-o-");
  161.  
  162. // Create start script, which will be used by the calling bash/cmd script.
  163. // build "rdmd --force %DFLAGS% -I%~dp0..\source -Jviews -Isource @deps.txt %LIBS% source\app.d" ~ application arguments
  164. // or with "/" instead of "\"
  165. Path exe_file_path;
  166. bool tmp_target = false;
  167. if (generate_binary) {
  168. if (settings.run && !isWritableDir(Path(buildsettings.targetPath), true)) {
  169. import std.random;
  170. auto rnd = to!string(uniform(uint.min, uint.max)) ~ "-";
  171. auto tmpdir = getTempDir()~".rdmd/source/";
  172. buildsettings.targetPath = tmpdir.toNativeString();
  173. buildsettings.targetName = rnd ~ buildsettings.targetName;
  174. m_temporaryFiles ~= tmpdir;
  175. tmp_target = true;
  176. }
  177. exe_file_path = Path(buildsettings.targetPath) ~ getTargetFileName(buildsettings, settings.platform);
  178. settings.compiler.setTarget(buildsettings, settings.platform);
  179. }
  180.  
  181. logDiagnostic("Application output name is '%s'", getTargetFileName(buildsettings, settings.platform));
  182.  
  183. string[] flags = ["--build-only", "--compiler="~settings.platform.compilerBinary];
  184. if (settings.force) flags ~= "--force";
  185. flags ~= buildsettings.dflags;
  186. flags ~= mainsrc.relativeTo(cwd).toNativeString();
  187.  
  188. if (buildsettings.preBuildCommands.length){
  189. logInfo("Running pre-build commands...");
  190. runCommands(buildsettings.preBuildCommands);
  191. }
  192.  
  193. logInfo("Building configuration "~config~", build type "~settings.buildType);
  194.  
  195. logInfo("Running rdmd...");
  196. logDiagnostic("rdmd %s", join(flags, " "));
  197. auto rdmd_pid = spawnProcess("rdmd" ~ flags);
  198. auto result = rdmd_pid.wait();
  199. enforce(result == 0, "Build command failed with exit code "~to!string(result));
  200.  
  201. if (tmp_target) {
  202. m_temporaryFiles ~= exe_file_path;
  203. foreach (f; buildsettings.copyFiles)
  204. m_temporaryFiles ~= Path(buildsettings.targetPath).parentPath ~ Path(f).head;
  205. }
  206. }
  207.  
  208. void performDirectBuild(GeneratorSettings settings, ref BuildSettings buildsettings, in Package pack, string config)
  209. {
  210. auto cwd = Path(getcwd());
  211.  
  212. auto generate_binary = !(buildsettings.options & BuildOptions.syntaxOnly);
  213. auto is_static_library = buildsettings.targetType == TargetType.staticLibrary || buildsettings.targetType == TargetType.library;
  214.  
  215. // make file paths relative to shrink the command line
  216. foreach (ref f; buildsettings.sourceFiles) {
  217. auto fp = Path(f);
  218. if( fp.absolute ) fp = fp.relativeTo(cwd);
  219. f = fp.toNativeString();
  220. }
  221.  
  222. logInfo("Building configuration \""~config~"\", build type "~settings.buildType);
  223.  
  224. // make all target/import paths relative
  225. string makeRelative(string path) { auto p = Path(path); if (p.absolute) p = p.relativeTo(cwd); return p.toNativeString(); }
  226. buildsettings.targetPath = makeRelative(buildsettings.targetPath);
  227. foreach (ref p; buildsettings.importPaths) p = makeRelative(p);
  228. foreach (ref p; buildsettings.stringImportPaths) p = makeRelative(p);
  229.  
  230. Path exe_file_path;
  231. bool is_temp_target = false;
  232. if (generate_binary) {
  233. if (settings.run && !isWritableDir(Path(buildsettings.targetPath), true)) {
  234. import std.random;
  235. auto rnd = to!string(uniform(uint.min, uint.max));
  236. auto tmppath = getTempDir()~("dub/"~rnd~"/");
  237. buildsettings.targetPath = tmppath.toNativeString();
  238. m_temporaryFiles ~= tmppath;
  239. is_temp_target = true;
  240. }
  241. exe_file_path = Path(buildsettings.targetPath) ~ getTargetFileName(buildsettings, settings.platform);
  242. }
  243.  
  244. if( buildsettings.preBuildCommands.length ){
  245. logInfo("Running pre-build commands...");
  246. runBuildCommands(buildsettings.preBuildCommands, buildsettings);
  247. }
  248.  
  249. buildWithCompiler(settings, buildsettings);
  250.  
  251. if (is_temp_target) {
  252. m_temporaryFiles ~= exe_file_path;
  253. foreach (f; buildsettings.copyFiles)
  254. m_temporaryFiles ~= Path(buildsettings.targetPath).parentPath ~ Path(f).head;
  255. }
  256. }
  257.  
  258. private string computeBuildID(string config, in BuildSettings buildsettings, GeneratorSettings settings)
  259. {
  260. import std.digest.digest;
  261. import std.digest.md;
  262. MD5 hash;
  263. hash.start();
  264. void addHash(in string[] strings...) { foreach (s; strings) { hash.put(cast(ubyte[])s); hash.put(0); } hash.put(0); }
  265. addHash(buildsettings.versions);
  266. addHash(buildsettings.debugVersions);
  267. //addHash(buildsettings.versionLevel);
  268. //addHash(buildsettings.debugLevel);
  269. addHash(buildsettings.dflags);
  270. addHash(buildsettings.lflags);
  271. addHash((cast(uint)buildsettings.options).to!string);
  272. addHash(buildsettings.stringImportPaths);
  273. addHash(settings.platform.architecture);
  274. addHash(settings.platform.compiler);
  275. //addHash(settings.platform.frontendVersion);
  276. auto hashstr = hash.finish().toHexString().idup;
  277.  
  278. return format("%s-%s-%s-%s-%s-%s", config, settings.buildType,
  279. settings.platform.platform.join("."),
  280. settings.platform.architecture.join("."),
  281. settings.platform.compilerBinary, hashstr);
  282. }
  283.  
  284. private void copyTargetFile(Path build_path, BuildSettings buildsettings, BuildPlatform platform)
  285. {
  286. auto filename = getTargetFileName(buildsettings, platform);
  287. auto src = build_path ~ filename;
  288. logDiagnostic("Copying target from %s to %s", src.toNativeString(), buildsettings.targetPath);
  289. copyFile(src, Path(buildsettings.targetPath) ~ filename, true);
  290. }
  291.  
  292. private bool isUpToDate(Path target_path, BuildSettings buildsettings, BuildPlatform platform)
  293. {
  294. import std.datetime;
  295.  
  296. auto targetfile = target_path ~ getTargetFileName(buildsettings, platform);
  297. if (!existsFile(targetfile)) {
  298. logDiagnostic("Target '%s' doesn't exist, need rebuild.", targetfile.toNativeString());
  299. return false;
  300. }
  301. auto targettime = getFileInfo(targetfile).timeModified;
  302.  
  303. auto allfiles = appender!(string[]);
  304. allfiles ~= buildsettings.sourceFiles;
  305. allfiles ~= buildsettings.importFiles;
  306. allfiles ~= buildsettings.stringImportFiles;
  307. // TODO: add library files
  308. /*foreach (p; m_project.getTopologicalPackageList())
  309. allfiles ~= p.packageInfoFile.toNativeString();*/
  310.  
  311. foreach (file; allfiles.data) {
  312. auto ftime = getFileInfo(file).timeModified;
  313. if (ftime > Clock.currTime)
  314. logWarn("File '%s' was modified in the future. Please re-save.", file);
  315. if (ftime > targettime) {
  316. logDiagnostic("File '%s' modified, need rebuild.", file);
  317. return false;
  318. }
  319. }
  320. return true;
  321. }
  322.  
  323. void buildWithCompiler(GeneratorSettings settings, BuildSettings buildsettings)
  324. {
  325. auto generate_binary = !(buildsettings.options & BuildOptions.syntaxOnly);
  326. auto is_static_library = buildsettings.targetType == TargetType.staticLibrary || buildsettings.targetType == TargetType.library;
  327.  
  328. Path target_file;
  329. scope (failure) {
  330. logInfo("FAIL %s %s %s" , buildsettings.targetPath, buildsettings.targetName, buildsettings.targetType);
  331. auto tpath = Path(buildsettings.targetPath) ~ getTargetFileName(buildsettings, settings.platform);
  332. if (generate_binary && existsFile(tpath))
  333. removeFile(tpath);
  334. }
  335.  
  336. /*
  337. NOTE: for DMD experimental separate compile/link is used, but this is not yet implemented
  338. on the other compilers. Later this should be integrated somehow in the build process
  339. (either in the package.json, or using a command line flag)
  340. */
  341. if (settings.platform.compilerBinary != "dmd" || !generate_binary || is_static_library) {
  342. // setup for command line
  343. if (generate_binary) settings.compiler.setTarget(buildsettings, settings.platform);
  344. settings.compiler.prepareBuildSettings(buildsettings, BuildSetting.commandLine);
  345.  
  346. // don't include symbols of dependencies (will be included by the top level target)
  347. if (is_static_library) buildsettings.sourceFiles = buildsettings.sourceFiles.filter!(f => !f.isLinkerFile()).array;
  348.  
  349. // invoke the compiler
  350. logInfo("Running %s...", settings.platform.compilerBinary);
  351. settings.compiler.invoke(buildsettings, settings.platform, settings.compileCallback);
  352. } else {
  353. // determine path for the temporary object file
  354. string tempobjname = buildsettings.targetName;
  355. version(Windows) tempobjname ~= ".obj";
  356. else tempobjname ~= ".o";
  357. Path tempobj = Path(buildsettings.targetPath) ~ tempobjname;
  358.  
  359. // setup linker command line
  360. auto lbuildsettings = buildsettings;
  361. lbuildsettings.sourceFiles = lbuildsettings.sourceFiles.filter!(f => isLinkerFile(f)).array;
  362. settings.compiler.setTarget(lbuildsettings, settings.platform);
  363. settings.compiler.prepareBuildSettings(lbuildsettings, BuildSetting.commandLineSeparate|BuildSetting.sourceFiles);
  364.  
  365. // setup compiler command line
  366. buildsettings.libs = null;
  367. buildsettings.lflags = null;
  368. buildsettings.addDFlags("-c", "-of"~tempobj.toNativeString());
  369. buildsettings.sourceFiles = buildsettings.sourceFiles.filter!(f => !isLinkerFile(f)).array;
  370. settings.compiler.prepareBuildSettings(buildsettings, BuildSetting.commandLine);
  371.  
  372. logInfo("Compiling...");
  373. settings.compiler.invoke(buildsettings, settings.platform, settings.compileCallback);
  374.  
  375. logInfo("Linking...");
  376. settings.compiler.invokeLinker(lbuildsettings, settings.platform, [tempobj.toNativeString()], settings.linkCallback);
  377. }
  378. }
  379.  
  380. void runTarget(Path exe_file_path, in BuildSettings buildsettings, string[] run_args, GeneratorSettings settings)
  381. {
  382. if (buildsettings.targetType == TargetType.executable) {
  383. auto cwd = Path(getcwd());
  384. auto runcwd = cwd;
  385. if (buildsettings.workingDirectory.length) {
  386. runcwd = Path(buildsettings.workingDirectory);
  387. if (!runcwd.absolute) runcwd = cwd ~ runcwd;
  388. logDiagnostic("Switching to %s", runcwd.toNativeString());
  389. chdir(runcwd.toNativeString());
  390. }
  391. scope(exit) chdir(cwd.toNativeString());
  392. if (!exe_file_path.absolute) exe_file_path = cwd ~ exe_file_path;
  393. auto exe_path_string = exe_file_path.relativeTo(runcwd).toNativeString();
  394. version (Posix) {
  395. if (!exe_path_string.startsWith(".") && !exe_path_string.startsWith("/"))
  396. exe_path_string = "./" ~ exe_path_string;
  397. }
  398. version (Windows) {
  399. if (!exe_path_string.startsWith(".") && (exe_path_string.length < 2 || exe_path_string[1] != ':'))
  400. exe_path_string = ".\\" ~ exe_path_string;
  401. }
  402. logInfo("Running %s %s", exe_path_string, run_args.join(" "));
  403. if (settings.runCallback) {
  404. auto res = execute(exe_path_string ~ run_args);
  405. settings.runCallback(res.status, res.output);
  406. } else {
  407. auto prg_pid = spawnProcess(exe_path_string ~ run_args);
  408. auto result = prg_pid.wait();
  409. enforce(result == 0, "Program exited with code "~to!string(result));
  410. }
  411. } else logInfo("Target is a library. Skipping execution.");
  412. }
  413.  
  414. void cleanupTemporaries()
  415. {
  416. foreach_reverse (f; m_temporaryFiles) {
  417. try {
  418. if (f.endsWithSlash) rmdir(f.toNativeString());
  419. else remove(f.toNativeString());
  420. } catch (Exception e) {
  421. logWarn("Failed to remove temporary file '%s': %s", f.toNativeString(), e.msg);
  422. logDiagnostic("Full error: %s", e.toString().sanitize);
  423. }
  424. }
  425. m_temporaryFiles = null;
  426. }
  427. }
  428.  
  429. private Path getMainSourceFile(in Package prj)
  430. {
  431. foreach (f; ["source/app.d", "src/app.d", "source/"~prj.name~".d", "src/"~prj.name~".d"])
  432. if (existsFile(prj.path ~ f))
  433. return prj.path ~ f;
  434. return prj.path ~ "source/app.d";
  435. }
  436.  
  437. unittest {
  438. version (Windows) {
  439. assert(isLinkerFile("test.obj"));
  440. assert(isLinkerFile("test.lib"));
  441. assert(isLinkerFile("test.res"));
  442. assert(!isLinkerFile("test.o"));
  443. assert(!isLinkerFile("test.d"));
  444. } else {
  445. assert(isLinkerFile("test.o"));
  446. assert(isLinkerFile("test.a"));
  447. assert(isLinkerFile("test.so"));
  448. assert(isLinkerFile("test.dylib"));
  449. assert(!isLinkerFile("test.obj"));
  450. assert(!isLinkerFile("test.d"));
  451. }
  452. }