diff --git a/changelog/build-cache.dd b/changelog/build-cache.dd new file mode 100644 index 0000000..a6ee0b7 --- /dev/null +++ b/changelog/build-cache.dd @@ -0,0 +1,14 @@ +Binary output will now be in a central cache + +Up until now, dub would output build artifact in the package directory. + +This allowed reuse of build artifact for dependencies, but also created +issues with large amount of build artifacts in the packages folder, +preventing the use of read-only location to store packages, +and making garbage collection of build artifacts unreliable. + +Starting from this version, build artifacts will be output by default to +`$HOME/.dub/cache/build/$BASE_PACKAGE_NAME/$PACKAGE_VERSION/[+$SUB_PACKAGE_NAME]` +on Linux, and +`%APPDATA%/cache/build/$BASE_PACKAGE_NAME/$PACKAGE_VERSION/[+$SUB_PACKAGE_NAME]` +on Windows. diff --git a/changelog/metadata-cache.dd b/changelog/metadata-cache.dd new file mode 100644 index 0000000..69cd539 --- /dev/null +++ b/changelog/metadata-cache.dd @@ -0,0 +1,9 @@ +DUB API breaking change: `Package.metadataCache` setter and getter have been removed + +Those two functions were used to provide access to the metadata cache file +to the generator. They were never intended for public consumption, +and the JSON file format was not stable. + +Due to the introduction of the build cache, they needed to be removed, +as there was no way to provide a sensible transition path, and they should be unused. +If you have a use case for it, please open an issue in dub repository. \ No newline at end of file diff --git a/source/dub/commandline.d b/source/dub/commandline.d index 95060b9..2651cba 100644 --- a/source/dub/commandline.d +++ b/source/dub/commandline.d @@ -1629,6 +1629,7 @@ GeneratorSettings settings = this.baseSettings; if (!settings.config.length) settings.config = m_defaultConfig; + settings.cache = dub.cachePathDontUse(); // See function's description // Ignore other options settings.buildSettings.options = this.baseSettings.buildSettings.options & BuildOption.lowmem; @@ -1688,27 +1689,10 @@ enforce(free_args.length == 0, "Cleaning a specific package isn't possible right now."); if (m_allPackages) { - bool any_error = false; - - foreach (p; dub.packageManager.getPackageIterator()) { - try dub.cleanPackage(p.path); - catch (Exception e) { - logWarn("Failed to clean package %s at %s: %s", p.name, p.path, e.msg); - any_error = true; - } - - foreach (sp; p.subPackages.filter!(sp => !sp.path.empty)) { - try dub.cleanPackage(p.path ~ sp.path); - catch (Exception e) { - logWarn("Failed to clean sub package of %s at %s: %s", p.name, p.path ~ sp.path, e.msg); - any_error = true; - } - } - } - - if (any_error) return 2; + dub.clean(); } else { - dub.cleanPackage(dub.rootPath); + dub.loadPackage(); + dub.clean(dub.project.rootPackage); } return 0; diff --git a/source/dub/dub.d b/source/dub/dub.d index 5d06304..24fd4de 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -255,6 +255,7 @@ m_dirs.userSettings = NativePath(dubHome); m_dirs.userPackages = m_dirs.userSettings; + m_dirs.cache = m_dirs.userPackages ~ "cache"; } } @@ -286,6 +287,7 @@ // config loaded from user settings and per-package config as well. if (!overrideDubHomeFromEnv && this.m_config.dubHome.set) { m_dirs.userPackages = NativePath(this.m_config.dubHome.expandEnvironmentVariables); + m_dirs.cache = m_dirs.userPackages ~ "cache"; } } @@ -683,6 +685,7 @@ */ void generateProject(string ide, GeneratorSettings settings) { + settings.cache = this.m_dirs.cache; // With a requested `unittest` config, switch to the special test runner // config (which doesn't require an existing `unittest` configuration). if (settings.config == "unittest") { @@ -701,6 +704,7 @@ */ void testProject(GeneratorSettings settings, string config, NativePath custom_main_file) { + settings.cache = this.m_dirs.cache; if (!custom_main_file.empty && !custom_main_file.absolute) custom_main_file = getWorkingDirectory() ~ custom_main_file; const test_config = m_project.addTestRunnerConfiguration(settings, !m_dryRun, config, custom_main_file); @@ -778,23 +782,36 @@ if (delimiter != "\0\0") writeln(); } - /// Cleans intermediate/cache files of the given package + /// Cleans intermediate/cache files of the given package (or all packages) + deprecated("Use `clean(Package)` instead") void cleanPackage(NativePath path) { - logInfo("Cleaning", Color.green, "package at %s", path.toNativeString().color(Mode.bold)); - enforce(!Package.findPackageFile(path).empty, "No package found.", path.toNativeString()); + auto ppack = Package.findPackageFile(path); + enforce(!ppack.empty, "No package found.", path.toNativeString()); + this.clean(Package.load(path, ppack)); + } + + /// Ditto + void clean() + { + const cache = this.m_dirs.cache; + logInfo("Cleaning", Color.green, "all artifacts at %s", + cache.toNativeString().color(Mode.bold)); + if (existsFile(cache)) + rmdirRecurse(cache.toNativeString()); + } + + /// Ditto + void clean(Package pack) + { + const cache = this.packageCache(pack); + logInfo("Cleaning", Color.green, "artifacts for package %s at %s", + pack.name.color(Mode.bold), + cache.toNativeString().color(Mode.bold)); // TODO: clear target files and copy files - - if (existsFile(path ~ ".dub/build")) rmdirRecurse((path ~ ".dub/build").toNativeString()); - if (existsFile(path ~ ".dub/metadata_cache.json")) removeFile((path ~ ".dub/metadata_cache.json")); - - auto p = Package.load(path); - if (p.getBuildSettings().targetType == TargetType.none) { - foreach (sp; p.subPackages.filter!(sp => !sp.path.empty)) { - cleanPackage(path ~ sp.path); - } - } + if (existsFile(cache)) + rmdirRecurse(cache.toNativeString()); } /// Fetches the package matching the dependency and places it in the specified location. @@ -1301,6 +1318,25 @@ } } + /** + * Compute and returns the path were artifacts are stored + * + * Expose `dub.generator.generator : packageCache` with this instance's + * configured cache. + */ + protected NativePath packageCache (Package pkg) const + { + return .packageCache(this.m_dirs.cache, pkg); + } + + /// Exposed because `commandLine` replicates `generateProject` for `dub describe` + /// instead of treating it like a regular generator... Remove this once the + /// flaw is fixed, and don't add more calls to this function! + package(dub) NativePath cachePathDontUse () const @safe pure nothrow @nogc + { + return this.m_dirs.cache; + } + /// Make a `GeneratorSettings` suitable to generate tools (DDOC, DScanner, etc...) private GeneratorSettings makeAppSettings () const { @@ -1820,6 +1856,20 @@ */ NativePath userPackages; + /** + * Location at which build/generation artifact will be written + * + * All build artifacts are stored under a single build cache, + * which is usually located under `$HOME/.dub/cache/` on POSIX, + * and `%LOCALAPPDATA%/dub/cache` on Windows. + * + * Versions of dub prior to v1.31.0 used to store artifact under the + * project directory, but this led to issues with packages stored on + * read-only filesystem / location, and lingering artifacts scattered + * through the filesystem. + */ + NativePath cache; + /// Returns: An instance of `SpecialDirs` initialized from the environment public static SpecialDirs make () { import std.file : tempDir; @@ -1840,6 +1890,7 @@ result.userSettings = NativePath(getcwd()) ~ result.userSettings; result.userPackages = result.userSettings; } + result.cache = result.userPackages ~ "cache"; return result; } } diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 19a39b0..d0f295c 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -249,7 +249,8 @@ string packageName = pack.basePackage is null ? pack.name : pack.basePackage.name; m_tempTargetExecutablePath = target_path = getTempDir() ~ format(".dub/build/%s-%s/%s/", packageName, pack.version_, build_id); } - else target_path = pack.path ~ format(".dub/build/%s/", build_id); + else + target_path = packageCache(settings.cache, pack) ~ "build/" ~ build_id; if (!settings.force && isUpToDate(target_path, buildsettings, settings, pack, packages, additional_dep_files)) { logInfo("Up-to-date", Color.green, "%s %s: target for configuration [%s] is up to date.", diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index f3b261a..7f4fc90 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -12,7 +12,9 @@ import dub.generators.build; import dub.generators.sublimetext; import dub.generators.visuald; +import dub.internal.utils; import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.data.json; import dub.internal.vibecompat.inet.path; import dub.internal.logging; import dub.package_; @@ -654,7 +656,10 @@ auto srcs = chain(bs.sourceFiles, bs.importFiles, bs.stringImportFiles) .filter!(f => dexts.canFind(f.extension)).filter!exists; // try to load cached filters first - auto cache = ti.pack.metadataCache; + const cacheFilePath = packageCache(NativePath(ti.buildSettings.targetPath), ti.pack) + ~ "metadata_cache.json"; + enum silent_fail = true; + auto cache = jsonFromFile(cacheFilePath, silent_fail); try { auto cachedFilters = cache["versionFilters"]; @@ -713,7 +718,9 @@ "versions": Json(versionFilters.data.map!Json.array), "debugVersions": Json(debugVersionFilters.data.map!Json.array), ]; - ti.pack.metadataCache = cache; + enum create_if_missing = true; + if (isWritableDir(cacheFilePath.parentPath, create_if_missing)) + writeJsonFile(cacheFilePath, cache); } private static void mergeFromDependent(const scope ref BuildSettings parent, ref BuildSettings child) @@ -760,8 +767,38 @@ } } +/** + * Compute and returns the path were artifacts are stored for a given package + * + * Artifacts are usually stored in: + * `$DUB_HOME/cache/$PKG_NAME/$PKG_VERSION[/+$SUB_PKG_NAME]/` + * Note that the leading `+` in the subpackage name is to avoid any ambiguity. + * Build artifacts are usually stored in a subfolder named "build", + * as their names are based on user-supplied values. + * + * Params: + * cachePath = Base path at which the build cache is located, + * e.g. `$HOME/.dub/cache/` + * pkg = The package. Cannot be `null`. + */ +package(dub) NativePath packageCache(NativePath cachePath, in Package pkg) +{ + import std.algorithm.searching : findSplit; + + assert(pkg !is null); + assert(!cachePath.empty); + + // For subpackages + if (const names = pkg.name.findSplit(":")) + return cachePath ~ names[0] ~ pkg.version_.toString() + ~ ("+" ~ names[2]); + // For regular packages + return cachePath ~ pkg.name ~ pkg.version_.toString(); +} + struct GeneratorSettings { + NativePath cache; BuildPlatform platform; Compiler compiler; string config; diff --git a/source/dub/package_.d b/source/dub/package_.d index 522425c..c406925 100644 --- a/source/dub/package_.d +++ b/source/dub/package_.d @@ -309,22 +309,6 @@ writeJsonFile(filename, m_info.toJson()); } - /// Get the metadata cache for this package - @property Json metadataCache() - { - enum silent_fail = true; - return jsonFromFile(m_path ~ ".dub/metadata_cache.json", silent_fail); - } - - /// Write metadata cache for this package - @property void metadataCache(Json json) - { - enum create_if_missing = true; - if (isWritableDir(m_path ~ ".dub", create_if_missing)) - writeJsonFile(m_path ~ ".dub/metadata_cache.json", json); - // TODO: store elsewhere - } - /** Returns the package recipe of a non-path-based sub package. For sub packages that are declared within the package recipe of the diff --git a/source/dub/project.d b/source/dub/project.d index b3ac03c..84027d7 100644 --- a/source/dub/project.d +++ b/source/dub/project.d @@ -320,7 +320,9 @@ mainfile = getTempFile("dub_test_root", ".d"); else { import dub.generators.build : computeBuildName; - mainfile = rootPackage.path ~ format(".dub/code/%s/dub_test_root.d", computeBuildName(config, settings, import_modules)); + mainfile = packageCache(settings.cache, this.rootPackage) ~ + format("code/%s/dub_test_root.d", + computeBuildName(config, settings, import_modules)); } auto escapedMainFile = mainfile.toNativeString().replace("$", "$$"); diff --git a/test/cache-generated-test-config.sh b/test/cache-generated-test-config.sh index a8ce0cc..b3e1721 100755 --- a/test/cache-generated-test-config.sh +++ b/test/cache-generated-test-config.sh @@ -2,7 +2,8 @@ . $(dirname "${BASH_SOURCE[0]}")/common.sh cd ${CURR_DIR}/cache-generated-test-config -rm -rf .dub +rm -rf $HOME/.dub/cache/cache-generated-test-config/ +DUB_CODE_CACHE_PATH="$HOME/.dub/cache/cache-generated-test-config/~master/code/" ## default test ${DUB} test --compiler=${DC} @@ -12,15 +13,15 @@ EXECUTABLE_TIME="$(${STAT} cache-generated-test-config-test-library)" [ -z "$EXECUTABLE_TIME" ] && die $LINENO 'no EXECUTABLE_TIME was found' -MAIN_TIME="$(${STAT} "$(ls .dub/code/*/dub_test_root.d)")" +MAIN_TIME="$(${STAT} "$(ls $DUB_CODE_CACHE_PATH/*/dub_test_root.d)")" [ -z "$MAIN_TIME" ] && die $LINENO 'no MAIN_TIME was found' ${DUB} test --compiler=${DC} -MAIN_FILES_COUNT=$(ls .dub/code/*/dub_test_root.d | wc -l) +MAIN_FILES_COUNT=$(ls $DUB_CODE_CACHE_PATH/*/dub_test_root.d | wc -l) [ $MAIN_FILES_COUNT -ne 1 ] && die $LINENO 'DUB generated more then one main file' [ "$EXECUTABLE_TIME" != "$(${STAT} cache-generated-test-config-test-library)" ] && die $LINENO 'The executable has been rebuilt' -[ "$MAIN_TIME" != "$(${STAT} "$(ls .dub/code/*/dub_test_root.d | head -n1)")" ] && die $LINENO 'The test main file has been rebuilt' +[ "$MAIN_TIME" != "$(${STAT} "$(ls $DUB_CODE_CACHE_PATH/*/dub_test_root.d | head -n1)")" ] && die $LINENO 'The test main file has been rebuilt' ## test with empty DFLAGS environment variable DFLAGS="" ${DUB} test --compiler=${DC} @@ -30,15 +31,15 @@ EXECUTABLE_TIME="$(${STAT} cache-generated-test-config-test-library)" [ -z "$EXECUTABLE_TIME" ] && die $LINENO 'no EXECUTABLE_TIME was found' -MAIN_TIME="$(${STAT} "$(ls .dub/code/*-\$DFLAGS-*/dub_test_root.d)")" +MAIN_TIME="$(${STAT} "$(ls $DUB_CODE_CACHE_PATH/*-\$DFLAGS-*/dub_test_root.d)")" [ -z "$MAIN_TIME" ] && die $LINENO 'no MAIN_TIME was found' DFLAGS="" ${DUB} test --compiler=${DC} -MAIN_FILES_COUNT=$(ls .dub/code/*-\$DFLAGS-*/dub_test_root.d | wc -l) +MAIN_FILES_COUNT=$(ls $DUB_CODE_CACHE_PATH/*-\$DFLAGS-*/dub_test_root.d | wc -l) [ $MAIN_FILES_COUNT -ne 1 ] && die $LINENO 'DUB generated more then one main file' [ "$EXECUTABLE_TIME" != "$(${STAT} cache-generated-test-config-test-library)" ] && die $LINENO 'The executable has been rebuilt' -[ "$MAIN_TIME" != "$(${STAT} "$(ls .dub/code/*-\$DFLAGS-*/dub_test_root.d | head -n1)")" ] && die $LINENO 'The test main file has been rebuilt' +[ "$MAIN_TIME" != "$(${STAT} "$(ls $DUB_CODE_CACHE_PATH/*-\$DFLAGS-*/dub_test_root.d | head -n1)")" ] && die $LINENO 'The test main file has been rebuilt' ## test with DFLAGS environment variable DFLAGS="-g" ${DUB} test --compiler=${DC} @@ -48,16 +49,16 @@ EXECUTABLE_TIME="$(${STAT} cache-generated-test-config-test-library)" [ -z "$EXECUTABLE_TIME" ] && die $LINENO 'no EXECUTABLE_TIME was found' -MAIN_TIME="$(${STAT} "$(ls .dub/code/*-\$DFLAGS-*/dub_test_root.d)")" +MAIN_TIME="$(${STAT} "$(ls $DUB_CODE_CACHE_PATH/*-\$DFLAGS-*/dub_test_root.d)")" [ -z "$MAIN_TIME" ] && die $LINENO 'no MAIN_TIME was found' DFLAGS="-g" ${DUB} test --compiler=${DC} -MAIN_FILES_COUNT=$(ls .dub/code/*-\$DFLAGS-*/dub_test_root.d | wc -l) +MAIN_FILES_COUNT=$(ls $DUB_CODE_CACHE_PATH/*-\$DFLAGS-*/dub_test_root.d | wc -l) [ $MAIN_FILES_COUNT -ne 1 ] && die $LINENO 'DUB generated more then one main file' [ "$EXECUTABLE_TIME" != "$(${STAT} cache-generated-test-config-test-library)" ] && die $LINENO 'The executable has been rebuilt' -[ "$MAIN_TIME" != "$(${STAT} "$(ls .dub/code/*-\$DFLAGS-*/dub_test_root.d | head -n1)")" ] && die $LINENO 'The test main file has been rebuilt' +[ "$MAIN_TIME" != "$(${STAT} "$(ls $DUB_CODE_CACHE_PATH/*-\$DFLAGS-*/dub_test_root.d | head -n1)")" ] && die $LINENO 'The test main file has been rebuilt' -exit 0 \ No newline at end of file +exit 0 diff --git a/test/issue97-targettype-none.sh b/test/issue97-targettype-none.sh index 3ffe303..9017321 100755 --- a/test/issue97-targettype-none.sh +++ b/test/issue97-targettype-none.sh @@ -3,7 +3,26 @@ ${DUB} build --root ${CURR_DIR}/issue97-targettype-none 2>&1 || true +BUILD_CACHE_A="$HOME/.dub/cache/issue97-targettype-none/~master/+a/build/" +BUILD_CACHE_B="$HOME/.dub/cache/issue97-targettype-none/~master/+b/build/" + +if [ ! -d $BUILD_CACHE_A ]; then + echo "Generated 'a' subpackage build artifact not found!" 1>&2 + exit 1 +fi +if [ ! -d $BUILD_CACHE_B ]; then + echo "Generated 'b' subpackage build artifact not found!" 1>&2 + exit 1 +fi + +${DUB} clean --root ${CURR_DIR}/issue97-targettype-none 2>&1 + # make sure both sub-packages are cleaned -OUTPUT=`${DUB} clean --root ${CURR_DIR}/issue97-targettype-none 2>&1` -echo $OUTPUT | grep -c "Cleaning package at .*/issue97-targettype-none/a/" > /dev/null -echo $OUTPUT | grep -c "Cleaning package at .*/issue97-targettype-none/b/" > /dev/null +if [ -d $BUILD_CACHE_A ]; then + echo "Generated 'a' subpackage build artifact were not cleaned!" 1>&2 + exit 1 +fi +if [ -d $BUILD_CACHE_B ]; then + echo "Generated 'b' subpackage build artifact were not cleaned!" 1>&2 + exit 1 +fi diff --git a/test/removed-dub-obj.sh b/test/removed-dub-obj.sh index 3686c61..8a168e2 100755 --- a/test/removed-dub-obj.sh +++ b/test/removed-dub-obj.sh @@ -2,14 +2,20 @@ . $(dirname "${BASH_SOURCE[0]}")/common.sh cd ${CURR_DIR}/removed-dub-obj -rm -rf .dub + +DUB_CACHE_PATH="$HOME/.dub/cache/removed-dub-obj/" + +rm -rf $DUB_CACHE_PATH ${DUB} build --compiler=${DC} -[ -d ".dub/obj" ] && die $LINENO '.dub/obj was found' +[ -d "$DUB_CACHE_PATH/obj" ] && die $LINENO "$DUB_CACHE_PATH/obj was found" if [[ ${DC} == *"ldc"* ]]; then - [ -f .dub/build/library-*ldc*/obj/test.o* ] || die $LINENO '.dub/build/library-*ldc*/obj/test.o* was not found' + if [ ! -f $DUB_CACHE_PATH/~master/build/library-*ldc*/obj/test.o* ]; then + ls -lR $DUB_CACHE_PATH + die $LINENO '$DUB_CACHE_PATH/~master/build/library-*ldc*/obj/test.o* was not found' + fi fi -exit 0 \ No newline at end of file +exit 0