diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff70485..a91f589 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -133,4 +133,5 @@ shell: bash - name: Codecov + if: matrix.do_test && runner.os != 'Windows' uses: codecov/codecov-action@v4 diff --git a/.github/workflows/pr_info_intro.yml b/.github/workflows/pr_info_intro.yml index a38f673..89b29fd 100644 --- a/.github/workflows/pr_info_intro.yml +++ b/.github/workflows/pr_info_intro.yml @@ -14,12 +14,10 @@ jobs: intro_comment: name: Make intro comment - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: 'Prepare sticky comment' - # commit of v2.5.0 - # same one used again at the bottom of the file to update the comment. - uses: marocchino/sticky-pull-request-comment@3d60a5b2dae89d44e0c6ddc69dd7536aec2071cd + uses: marocchino/sticky-pull-request-comment@v2.9.0 with: message: | Thanks for your Pull Request and making D better! diff --git a/.github/workflows/pr_info_post.yml b/.github/workflows/pr_info_post.yml index 6774e4f..2fc012b 100644 --- a/.github/workflows/pr_info_post.yml +++ b/.github/workflows/pr_info_post.yml @@ -12,17 +12,17 @@ jobs: comment: name: PR Info - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: > github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' steps: # from https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ - name: 'Download artifact' - uses: actions/github-script@v3.1.0 + uses: actions/github-script@v7 with: script: | - var artifacts = await github.actions.listWorkflowRunArtifacts({ + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, run_id: ${{github.event.workflow_run.id }}, @@ -30,7 +30,7 @@ var matchArtifact = artifacts.data.artifacts.filter((artifact) => { return artifact.name == "pr" })[0]; - var download = await github.actions.downloadArtifact({ + var download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, artifact_id: matchArtifact.id, @@ -46,7 +46,7 @@ echo "PR_ID=$PR_ID" >> $GITHUB_ENV - name: Update GitHub comment - uses: marocchino/sticky-pull-request-comment@3d60a5b2dae89d44e0c6ddc69dd7536aec2071cd + uses: marocchino/sticky-pull-request-comment@v2.9.0 with: path: ./comment.txt number: ${{ env.PR_ID }} diff --git a/.github/workflows/pr_info_untrusted.yml b/.github/workflows/pr_info_untrusted.yml index 56b8214..338511a 100644 --- a/.github/workflows/pr_info_untrusted.yml +++ b/.github/workflows/pr_info_untrusted.yml @@ -12,7 +12,7 @@ jobs: pr_info: name: PR Info - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: # we first create a comment thanking the user in pr_info_intro.yml # (separate step due to needing GITHUB_TOKEN access) diff --git a/build-files.txt b/build-files.txt index 3a9ec14..c9f011a 100644 --- a/build-files.txt +++ b/build-files.txt @@ -47,6 +47,8 @@ source/dub/internal/dyaml/tagdirective.d source/dub/internal/dyaml/token.d source/dub/internal/git.d +source/dub/internal/io/filesystem.d +source/dub/internal/io/realfs.d source/dub/internal/libInputVisitor.d source/dub/internal/sdlang/ast.d source/dub/internal/sdlang/exception.d diff --git a/build.d b/build.d index 03038f4..f2fde8a 100755 --- a/build.d +++ b/build.d @@ -4,12 +4,12 @@ Standalone build script for DUB This script can be called from anywhere, as it deduces absolute paths - based on the script's placement in the repository. + based on the script's location in the source tree. - Invoking it while making use of all the options would like like this: - DMD=ldmd2 DFLAGS="-O -inline" ./build.d my-dub-version - Using an environment variable for the version is also supported: - DMD=dmd DFLAGS="-w -g" GITVER="1.2.3" ./build.d + Invoking it while making use of all the options would look like this: + DMD=ldmd2 GITVER="1.2.3" ./build.d -O -inline + The GITVER environment variable determines the version of + the repository to build against. Copyright: D Language Foundation Authors: Mathias 'Geod24' Lang diff --git a/changelog/selections_from_parent_dir.dd b/changelog/selections_from_parent_dir.dd new file mode 100644 index 0000000..923d8e4 --- /dev/null +++ b/changelog/selections_from_parent_dir.dd @@ -0,0 +1,16 @@ +`dub.selections.json` files are now looked up in parent directories too + +In case the root package directory doesn't contain a `dub.selections.json` +file, dub now looks in parent directories too and potentially uses the +first (deepest) one it finds - if and only if that JSON file contains an +optional new `"inheritable": true` flag. + +This allows using a 'central' `dub.selections.json` file for a repository +containing multiple dub projects, making it automatically apply to all +builds in that source tree if located in the repository root directory +(unless a local `dub.selections.json` overrides it). + +Such an inherited selections file is never mutated when running dub for a +nested project, i.e., changes are always saved to a *local* +`dub.selections.json` file. E.g., when running `dub upgrade` for a nested +project. diff --git a/dub.selections.json b/dub.selections.json index 3a78158..b10e96e 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -4,18 +4,18 @@ "botan": "1.12.19", "botan-math": "1.0.3", "diet-ng": "1.8.1", - "eventcore": "0.9.27", + "eventcore": "0.9.30", "libasync": "0.8.6", "libev": "5.0.0+4.04", "libevent": "2.0.2+2.0.16", "memutils": "1.0.10", "mir-linux-kernel": "1.0.1", "openssl": "3.3.3", - "openssl-static": "1.0.2+3.0.8", + "openssl-static": "1.0.5+3.0.8", "stdx-allocator": "2.77.5", - "taggedalgebraic": "0.11.22", - "vibe-container": "1.0.1", - "vibe-core": "2.7.1", - "vibe-d": "0.9.7" + "taggedalgebraic": "0.11.23", + "vibe-container": "1.3.1", + "vibe-core": "2.8.5", + "vibe-d": "0.9.8" } } diff --git a/source/dub/commandline.d b/source/dub/commandline.d index 7bb40bf..575e189 100644 --- a/source/dub/commandline.d +++ b/source/dub/commandline.d @@ -568,11 +568,6 @@ SkipPackageSuppliers skipRegistry = SkipPackageSuppliers.default_; PlacementLocation placementLocation = PlacementLocation.user; - deprecated("Use `Color` instead, the previous naming was a limitation of error message formatting") - alias color = Color; - deprecated("Use `colorMode` instead") - alias color_mode = colorMode; - private void parseColor(string option, string value) @safe { // `automatic`, `on`, `off` are there for backwards compatibility @@ -878,10 +873,10 @@ // should simply retry over all registries instead of using a special // FallbackPackageSupplier. auto urls = url.splitter(' '); - PackageSupplier ps = getRegistryPackageSupplier(urls.front); + PackageSupplier ps = _getRegistryPackageSupplier(urls.front); urls.popFront; if (!urls.empty) - ps = new FallbackPackageSupplier(ps ~ urls.map!getRegistryPackageSupplier.array); + ps = new FallbackPackageSupplier(ps ~ urls.map!_getRegistryPackageSupplier.array); return ps; }) .array; diff --git a/source/dub/compilers/ldc.d b/source/dub/compilers/ldc.d index 3959df6..0ca58c2 100644 --- a/source/dub/compilers/ldc.d +++ b/source/dub/compilers/ldc.d @@ -104,7 +104,6 @@ void prepareBuildSettings(ref BuildSettings settings, const scope ref BuildPlatform platform, BuildSetting fields = BuildSetting.all) const { - import std.format : format; enforceBuildRequirements(settings); // Keep the current dflags at the end of the array so that they will overwrite other flags. @@ -118,9 +117,6 @@ settings.addDFlags(t[1]); } - // since LDC always outputs multiple object files, avoid conflicts by default - settings.addDFlags("--oq", format("-od=%s/obj", settings.targetPath)); - if (!(fields & BuildSetting.versions)) { settings.addDFlags(settings.versions.map!(s => "-d-version="~s)().array()); settings.versions = null; @@ -241,7 +237,17 @@ case TargetType.executable: break; case TargetType.library: case TargetType.staticLibrary: - settings.addDFlags("-lib"); + // -oq: name object files uniquely (so the files don't collide) + settings.addDFlags("-lib", "-oq"); + // -cleanup-obj (supported since LDC v1.1): remove object files after archiving to static lib + if (platform.frontendVersion >= 2071) { + settings.addDFlags("-cleanup-obj"); + } + if (platform.frontendVersion < 2095) { + // Since LDC v1.25, -cleanup-obj defaults to a unique temp -od directory + // We need to resort to a unique-ish -od directory before that + settings.addDFlags("-od=" ~ settings.targetPath ~ "/obj"); + } break; case TargetType.dynamicLibrary: settings.addDFlags("-shared"); diff --git a/source/dub/dependency.d b/source/dub/dependency.d index ddadb84..f7781e5 100644 --- a/source/dub/dependency.d +++ b/source/dub/dependency.d @@ -729,7 +729,10 @@ unittest { assert(VersionRange.fromString("~>1.0.4").toString() == "~>1.0.4"); assert(VersionRange.fromString("~>1.4").toString() == "~>1.4"); - assert(VersionRange.fromString("~>2").toString() == "~>2"); + // https://github.com/dlang/dub/issues/2830 + assert(VersionRange.fromString("~>2").toString() == "~>2.0"); + assert(VersionRange.fromString("~>5.0").toString() == "~>5.0"); + assert(VersionRange.fromString("~>1.0.4+1.2.3").toString() == "~>1.0.4"); assert(VersionRange.fromString("^0.1.2").toString() == "^0.1.2"); assert(VersionRange.fromString("^1.2.3").toString() == "^1.2.3"); @@ -1116,6 +1119,7 @@ string r; if (this == Invalid) return "no"; + if (this.matchesAny()) return "*"; if (this.isExactVersion() && m_inclusiveA && m_inclusiveB) { // Special "==" case if (m_versA == Version.masterBranch) return "~master"; @@ -1131,7 +1135,10 @@ auto parts = va.splitter('.').array; assert(parts.length == 3, "Version string with a digit group count != 3: "~va); - foreach (i; 0 .. 3) { + // Start at 1 because the notation `~>1` and `^1` are equivalent + // to `~>1.0` and `^1.0`, and the latter are better understood + // and recognized by users. See for example issue 2830. + foreach (i; 1 .. 3) { auto vp = parts[0 .. i+1].join("."); auto ve = Version(expandVersion(vp)); auto veb = Version(bumpVersion(vp) ~ "-0"); @@ -1142,9 +1149,12 @@ } } - if (m_versA != Version.minRelease) r = (m_inclusiveA ? ">=" : ">") ~ m_versA.toString(); - if (m_versB != Version.maxRelease) r ~= (r.length==0 ? "" : " ") ~ (m_inclusiveB ? "<=" : "<") ~ m_versB.toString(); - if (this.matchesAny()) r = ">=0.0.0"; + if (m_versA != Version.minRelease || !m_inclusiveA) + r = (m_inclusiveA ? ">=" : ">") ~ m_versA.toString(); + if (m_versB != Version.maxRelease || !m_inclusiveB) + r ~= (r.length == 0 ? "" : " ") ~ (m_inclusiveB ? "<=" : "<") ~ + m_versB.toString(); + return r; } @@ -1241,6 +1251,13 @@ assert(Version("1.0.0+foo").matches(Version("1.0.0+foo"), VersionMatchMode.strict)); } +// Erased version specification for dependency, converted to "" instead of ">0.0.0" +// https://github.com/dlang/dub/issues/2901 +unittest +{ + assert(VersionRange.fromString(">0.0.0").toString() == ">0.0.0"); +} + /// Determines whether the given string is a Git hash. bool isGitHash(string hash) @nogc nothrow pure @safe { diff --git a/source/dub/dub.d b/source/dub/dub.d index ec8e83f..75f268b 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -77,7 +77,10 @@ // Private to avoid a bug in `defaultPackageSuppliers` with `map` triggering a deprecation // even though the context is deprecated. -private PackageSupplier _getRegistryPackageSupplier(string url) +// Also used from `commandline`. Note that this is replaced by a method +// in the `Dub` class, to allow for proper dependency injection, +// but `commandline` is currently completely excluded. +package(dub) PackageSupplier _getRegistryPackageSupplier(string url) { switch (url.startsWith("dub+", "mvn+", "file://")) { @@ -373,15 +376,15 @@ import dub.test.base : TestDub; scope (exit) environment.remove("DUB_REGISTRY"); - auto dub = new TestDub(null, ".", null, SkipPackageSuppliers.configured); + auto dub = new TestDub(null, "/dub/project/", null, SkipPackageSuppliers.configured); assert(dub.packageSuppliers.length == 0); environment["DUB_REGISTRY"] = "http://example.com/"; - dub = new TestDub(null, ".", null, SkipPackageSuppliers.configured); + dub = new TestDub(null, "/dub/project/", null, SkipPackageSuppliers.configured); assert(dub.packageSuppliers.length == 1); environment["DUB_REGISTRY"] = "http://example.com/;http://foo.com/"; - dub = new TestDub(null, ".", null, SkipPackageSuppliers.configured); + dub = new TestDub(null, "/dub/project/", null, SkipPackageSuppliers.configured); assert(dub.packageSuppliers.length == 2); - dub = new TestDub(null, ".", [new RegistryPackageSupplier(URL("http://bar.com/"))], SkipPackageSuppliers.configured); + dub = new TestDub(null, "/dub/project/", [new RegistryPackageSupplier(URL("http://bar.com/"))], SkipPackageSuppliers.configured); assert(dub.packageSuppliers.length == 3); dub = new TestDub(); @@ -405,7 +408,7 @@ * Returns: * A new instance of a `PackageSupplier`. */ - protected PackageSupplier makePackageSupplier(string url) const + protected PackageSupplier makePackageSupplier(string url) { switch (url.startsWith("dub+", "mvn+", "file://")) { @@ -519,7 +522,7 @@ /// Loads a specific package as the main project package (can be a sub package) void loadPackage(Package pack) { - auto selections = Project.loadSelections(pack); + auto selections = Project.loadSelections(pack.path, m_packageManager); m_project = new Project(m_packageManager, pack, selections); } @@ -1649,12 +1652,12 @@ unittest { import dub.test.base : TestDub; - import std.path: buildPath, absolutePath; auto dub = new TestDub(null, ".", null, SkipPackageSuppliers.configured); + immutable testdir = getWorkingDirectory() ~ "test-determineDefaultCompiler"; + immutable olddc = environment.get("DC", null); immutable oldpath = environment.get("PATH", null); - immutable testdir = "test-determineDefaultCompiler"; void repairenv(string name, string var) { if (var !is null) @@ -1664,33 +1667,33 @@ } scope (exit) repairenv("DC", olddc); scope (exit) repairenv("PATH", oldpath); - scope (exit) rmdirRecurse(testdir); + scope (exit) std.file.rmdirRecurse(testdir.toNativeString()); version (Windows) enum sep = ";", exe = ".exe"; version (Posix) enum sep = ":", exe = ""; - immutable dmdpath = testdir.buildPath("dmd", "bin"); - immutable ldcpath = testdir.buildPath("ldc", "bin"); - mkdirRecurse(dmdpath); - mkdirRecurse(ldcpath); - immutable dmdbin = dmdpath.buildPath("dmd"~exe); - immutable ldcbin = ldcpath.buildPath("ldc2"~exe); - std.file.write(dmdbin, null); - std.file.write(ldcbin, null); + immutable dmdpath = testdir ~ "dmd" ~ "bin"; + immutable ldcpath = testdir ~ "ldc" ~ "bin"; + ensureDirectory(dmdpath); + ensureDirectory(ldcpath); + immutable dmdbin = dmdpath ~ ("dmd" ~ exe); + immutable ldcbin = ldcpath ~ ("ldc2" ~ exe); + writeFile(dmdbin, null); + writeFile(ldcbin, null); - environment["DC"] = dmdbin.absolutePath(); - assert(dub.determineDefaultCompiler() == dmdbin.absolutePath()); + environment["DC"] = dmdbin.toNativeString(); + assert(dub.determineDefaultCompiler() == dmdbin.toNativeString()); environment["DC"] = "dmd"; - environment["PATH"] = dmdpath ~ sep ~ ldcpath; + environment["PATH"] = dmdpath.toNativeString() ~ sep ~ ldcpath.toNativeString(); assert(dub.determineDefaultCompiler() == "dmd"); environment["DC"] = "ldc2"; - environment["PATH"] = dmdpath ~ sep ~ ldcpath; + environment["PATH"] = dmdpath.toNativeString() ~ sep ~ ldcpath.toNativeString(); assert(dub.determineDefaultCompiler() == "ldc2"); environment.remove("DC"); - environment["PATH"] = ldcpath ~ sep ~ dmdpath; + environment["PATH"] = ldcpath.toNativeString() ~ sep ~ dmdpath.toNativeString(); assert(dub.determineDefaultCompiler() == "ldc2"); } diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index c893fb0..d3cc65c 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -766,15 +766,17 @@ } unittest { // issue #1235 - pass no library files to compiler command line when building a static lib - import dub.internal.vibecompat.data.json : parseJsonString; + import dub.recipe.io : parsePackageRecipe; import dub.compilers.gdc : GDCCompiler; import dub.platform : determinePlatform; version (Windows) auto libfile = "bar.lib"; else auto libfile = "bar.a"; - auto desc = parseJsonString(`{"name": "test", "targetType": "library", "sourceFiles": ["foo.d", "`~libfile~`"]}`); - auto pack = new Package(desc, NativePath("/tmp/fooproject")); + auto recipe = parsePackageRecipe( + `{"name":"test", "targetType":"library", "sourceFiles":["foo.d", "`~libfile~`"]}`, + `/tmp/fooproject/dub.json`); + auto pack = new Package(recipe, NativePath("/tmp/fooproject")); auto pman = new PackageManager(pack.path, NativePath("/tmp/foo/"), NativePath("/tmp/foo/"), false); auto prj = new Project(pman, pack); diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index 7402438..c112cc4 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -1225,7 +1225,7 @@ version(Posix) { // https://github.com/dlang/dub/issues/2238 unittest { - import dub.internal.vibecompat.data.json : parseJsonString; + import dub.recipe.io : parsePackageRecipe; import dub.compilers.gdc : GDCCompiler; import std.algorithm : canFind; import std.path : absolutePath; @@ -1235,8 +1235,10 @@ write("dubtest/preGen/source/foo.d", ""); scope(exit) rmdirRecurse("dubtest"); - auto desc = parseJsonString(`{"name": "test", "targetType": "library", "preGenerateCommands": ["touch $PACKAGE_DIR/source/bar.d"]}`); - auto pack = new Package(desc, NativePath("dubtest/preGen".absolutePath)); + auto recipe = parsePackageRecipe( + `{"name":"test", "targetType":"library", "preGenerateCommands":["touch $PACKAGE_DIR/source/bar.d"]}`, + `dubtest/preGen/dub.json`); + auto pack = new Package(recipe, NativePath("dubtest/preGen".absolutePath)); auto pman = new PackageManager(pack.path, NativePath("/tmp/foo/"), NativePath("/tmp/foo/"), false); auto prj = new Project(pman, pack); diff --git a/source/dub/internal/configy/Read.d b/source/dub/internal/configy/Read.d index 71f877f..cb07be2 100644 --- a/source/dub/internal/configy/Read.d +++ b/source/dub/internal/configy/Read.d @@ -296,11 +296,14 @@ /// Ditto public Nullable!T parseConfigFileSimple (T) (in CLIArgs args, StrictMode strict = StrictMode.Error) { + return wrapException(parseConfigFile!T(args, strict)); +} + +/// Ditto +public Nullable!T wrapException (T) (lazy T parseCall) +{ try - { - Node root = Loader.fromFile(args.config_path).load(); - return nullable(parseConfig!T(args, root, strict)); - } + return nullable(parseCall); catch (ConfigException exc) { exc.printException(); diff --git a/source/dub/internal/io/filesystem.d b/source/dub/internal/io/filesystem.d new file mode 100644 index 0000000..4140055 --- /dev/null +++ b/source/dub/internal/io/filesystem.d @@ -0,0 +1,96 @@ +/** + * An abstract filesystem representation + * + * This interface allows to represent the file system to various part of Dub. + * Instead of direct use of `std.file`, an implementation of this interface can + * be used, allowing to mock all I/O in unittest on a thread-local basis. + */ +module dub.internal.io.filesystem; + +public import std.datetime.systime; + +public import dub.internal.vibecompat.inet.path; + +/// Ditto +public interface Filesystem +{ + static import dub.internal.vibecompat.core.file; + + /// TODO: Remove, the API should be improved + public alias IterateDirDg = int delegate( + scope int delegate(ref dub.internal.vibecompat.core.file.FileInfo)); + + /// Ditto + public IterateDirDg iterateDirectory (in NativePath path) scope; + + /// Returns: The `path` of this FSEntry + public abstract NativePath getcwd () const scope; + + /** + * Implements `mkdir -p`: Create a directory and every intermediary + * + * There is no way to error out on intermediate directory, + * like standard mkdir does. If you want this behavior, + * simply check (`existsDirectory`) if the parent directory exists. + * + * Params: + * path = The path of the directory to be created. + */ + public abstract void mkdir (in NativePath path) scope; + + /// Checks the existence of a file + public abstract bool existsFile (in NativePath path) const scope; + + /// Checks the existence of a directory + public abstract bool existsDirectory (in NativePath path) const scope; + + /// Reads a file, returns the content as `ubyte[]` + public abstract ubyte[] readFile (in NativePath path) const scope; + + /// Reads a file, returns the content as text + public abstract string readText (in NativePath path) const scope; + + /// Write to this file + public final void writeFile (in NativePath path, const(char)[] data) scope + { + import std.string : representation; + + this.writeFile(path, data.representation); + } + + /// Ditto + public abstract void writeFile (in NativePath path, const(ubyte)[] data) scope; + + /** Remove a file + * + * Always error if the target is a directory. + * Does not error if the target does not exists + * and `force` is set to `true`. + * + * Params: + * path = Path to the file to remove + * force = Whether to ignore non-existing file, + * default to `false`. + */ + public void removeFile (in NativePath path, bool force = false); + + /** Remove a directory + * + * Remove an existing empty directory. + * If `force` is set to `true`, no error will be thrown + * if the directory is empty or non-existing. + * + * Params: + * path = Path to the directory to remove + * force = Whether to ignore non-existing / non-empty directories, + * default to `false`. + */ + public void removeDir (in NativePath path, bool force = false); + + /// Implement `std.file.setTimes` + public void setTimes (in NativePath path, in SysTime accessTime, + in SysTime modificationTime); + + /// Implement `std.file.setAttributes` + public void setAttributes (in NativePath path, uint attributes); +} diff --git a/source/dub/internal/io/mockfs.d b/source/dub/internal/io/mockfs.d new file mode 100644 index 0000000..4c0f3d6 --- /dev/null +++ b/source/dub/internal/io/mockfs.d @@ -0,0 +1,491 @@ +/******************************************************************************* + + An unittest implementation of `Filesystem` + +*******************************************************************************/ + +module dub.internal.io.mockfs; + +public import dub.internal.io.filesystem; + +static import dub.internal.vibecompat.core.file; + +import std.algorithm; +import std.exception; +import std.range; +import std.string; + +/// Ditto +public final class MockFS : Filesystem { + /// + private FSEntry cwd; + + /// + public this () scope + { + this.cwd = new FSEntry(); + } + + public override NativePath getcwd () const scope + { + return this.cwd.path(); + } + + /// + public override bool existsDirectory (in NativePath path) const scope + { + auto entry = this.cwd.lookup(path); + return entry !is null && entry.isDirectory(); + } + + /// Ditto + public override void mkdir (in NativePath path) scope + { + this.cwd.mkdir(path); + } + + /// Ditto + public override bool existsFile (in NativePath path) const scope + { + auto entry = this.cwd.lookup(path); + return entry !is null && entry.isFile(); + } + + /// Ditto + public override void writeFile (in NativePath path, const(ubyte)[] data) + scope + { + return this.cwd.writeFile(path, data); + } + + /// Reads a file, returns the content as `ubyte[]` + public override ubyte[] readFile (in NativePath path) const scope + { + return this.cwd.readFile(path); + } + + /// Ditto + public override string readText (in NativePath path) const scope + { + return this.cwd.readText(path); + } + + /// Ditto + public override IterateDirDg iterateDirectory (in NativePath path) scope + { + enforce(this.existsDirectory(path), + path.toNativeString() ~ " does not exists or is not a directory"); + auto dir = this.cwd.lookup(path); + int iterator(scope int delegate(ref dub.internal.vibecompat.core.file.FileInfo) del) { + foreach (c; dir.children) { + dub.internal.vibecompat.core.file.FileInfo fi; + fi.name = c.name; + fi.timeModified = c.attributes.modification; + final switch (c.attributes.type) { + case FSEntry.Type.File: + fi.size = c.content.length; + break; + case FSEntry.Type.Directory: + fi.isDirectory = true; + break; + } + if (auto res = del(fi)) + return res; + } + return 0; + } + return &iterator; + } + + /// Ditto + public override void removeFile (in NativePath path, bool force = false) scope + { + return this.cwd.removeFile(path); + } + + /// + public override void removeDir (in NativePath path, bool force = false) + { + this.cwd.removeDir(path, force); + } + + /// Ditto + public override void setTimes (in NativePath path, in SysTime accessTime, + in SysTime modificationTime) + { + auto e = this.cwd.lookup(path); + enforce(e !is null, + "setTimes: No such file or directory: " ~ path.toNativeString()); + e.setTimes(accessTime, modificationTime); + } + + /// Ditto + public override void setAttributes (in NativePath path, uint attributes) + { + auto e = this.cwd.lookup(path); + enforce(e !is null, + "setAttributes: No such file or directory: " ~ path.toNativeString()); + e.setAttributes(attributes); + } + + /** + * Converts an `Filesystem` and its children to a `ZipFile` + */ + public ubyte[] serializeToZip (string rootPath) { + import std.path; + import std.zip; + + scope z = new ZipArchive(); + void addToZip(scope string dir, scope FSEntry e) { + auto m = new ArchiveMember(); + m.name = dir.buildPath(e.name); + m.fileAttributes = e.attributes.attrs; + m.time = e.attributes.modification; + + final switch (e.attributes.type) { + case FSEntry.Type.Directory: + // We need to ensure the directory entry ends with a slash + // otherwise it will be considered as a file. + if (m.name[$-1] != '/') + m.name ~= '/'; + z.addMember(m); + foreach (c; e.children) + addToZip(m.name, c); + break; + case FSEntry.Type.File: + m.expandedData = e.content; + z.addMember(m); + } + } + addToZip(rootPath, this.cwd); + return cast(ubyte[]) z.build(); + } +} + +/// The backing logic behind `MockFS` +public class FSEntry +{ + /// Type of file system entry + public enum Type : ubyte { + Directory, + File, + } + + /// List FSEntry attributes + protected struct Attributes { + /// The type of FSEntry, see `FSEntry.Type` + public Type type; + /// System-specific attributes for this `FSEntry` + public uint attrs; + /// Last access time + public SysTime access; + /// Last modification time + public SysTime modification; + } + /// Ditto + protected Attributes attributes; + + /// The name of this node + protected string name; + /// The parent of this entry (can be null for the root) + protected FSEntry parent; + union { + /// Children for this FSEntry (with type == Directory) + protected FSEntry[] children; + /// Content for this FDEntry (with type == File) + protected ubyte[] content; + } + + /// Creates a new FSEntry + package(dub) this (FSEntry p, Type t, string n) + { + // Avoid 'DOS File Times cannot hold dates prior to 1980.' exception + import std.datetime.date; + SysTime DefaultTime = SysTime(DateTime(2020, 01, 01)); + + this.attributes.type = t; + this.parent = p; + this.name = n; + this.attributes.access = DefaultTime; + this.attributes.modification = DefaultTime; + } + + /// Create the root of the filesystem, only usable from this module + package(dub) this () + { + import std.datetime.date; + SysTime DefaultTime = SysTime(DateTime(2020, 01, 01)); + + this.attributes.type = Type.Directory; + this.attributes.access = DefaultTime; + this.attributes.modification = DefaultTime; + } + + /// Get a direct children node, returns `null` if it can't be found + protected inout(FSEntry) lookup(string name) inout return scope + { + assert(!name.canFind('/')); + foreach (c; this.children) + if (c.name == name) + return c; + return null; + } + + /// Get an arbitrarily nested children node + protected inout(FSEntry) lookup(NativePath path) inout return scope + { + auto relp = this.relativePath(path); + relp.normalize(); // try to get rid of `..` + if (relp.empty) + return this; + auto segments = relp.bySegment; + if (auto c = this.lookup(segments.front.name)) { + segments.popFront(); + return !segments.empty ? c.lookup(NativePath(segments)) : c; + } + return null; + } + + /** Get the parent `FSEntry` of a `NativePath` + * + * If the parent doesn't exist, an `Exception` will be thrown + * unless `silent` is provided. If the parent path is a file, + * an `Exception` will be thrown regardless of `silent`. + * + * Params: + * path = The path to look up the parent for + * silent = Whether to error on non-existing parent, + * default to `false`. + */ + protected inout(FSEntry) getParent(NativePath path, bool silent = false) + inout return scope + { + // Relative path in the current directory + if (!path.hasParentPath()) + return this; + + // If we're not in the right `FSEntry`, recurse + const parentPath = path.parentPath(); + auto p = this.lookup(parentPath); + enforce(silent || p !is null, + "No such directory: " ~ parentPath.toNativeString()); + enforce(p is null || p.attributes.type == Type.Directory, + "Parent path is not a directory: " ~ parentPath.toNativeString()); + return p; + } + + /// Returns: A path relative to `this.path` + protected NativePath relativePath(NativePath path) const scope + { + assert(!path.absolute() || path.startsWith(this.path), + "Calling relativePath with a differently rooted path"); + return path.absolute() ? path.relativeTo(this.path) : path; + } + + /*+************************************************************************* + + Utility function + + Below this banners are functions that are provided for the convenience + of writing tests for `Dub`. + + ***************************************************************************/ + + /// Prints a visual representation of the filesystem to stdout for debugging + public void print(bool content = false) const scope + { + import std.range : repeat; + static import std.stdio; + + size_t indent; + for (auto p = &this.parent; (*p) !is null; p = &p.parent) + indent++; + // Don't print anything (even a newline) for root + if (this.parent is null) + std.stdio.write('/'); + else + std.stdio.write('|', '-'.repeat(indent), ' ', this.name, ' '); + + final switch (this.attributes.type) { + case Type.Directory: + std.stdio.writeln('(', this.children.length, " entries):"); + foreach (c; this.children) + c.print(content); + break; + case Type.File: + if (!content) + std.stdio.writeln('(', this.content.length, " bytes)"); + else if (this.name.endsWith(".json") || this.name.endsWith(".sdl")) + std.stdio.writeln('(', this.content.length, " bytes): ", + cast(string) this.content); + else + std.stdio.writeln('(', this.content.length, " bytes): ", + this.content); + break; + } + } + + /*+************************************************************************* + + Public filesystem functions + + Below this banners are functions which mimic the behavior of a file + system. + + ***************************************************************************/ + + /// Returns: The `path` of this FSEntry + public NativePath path () const scope + { + if (this.parent is null) + return NativePath("/"); + auto thisPath = this.parent.path ~ this.name; + thisPath.endsWithSlash = (this.attributes.type == Type.Directory); + return thisPath; + } + + /// Implements `mkdir -p`, returns the created directory + public FSEntry mkdir (in NativePath path) scope + { + auto relp = this.relativePath(path); + // Check if the child already exists + auto segments = relp.bySegment; + auto child = this.lookup(segments.front.name); + if (child is null) { + child = new FSEntry(this, Type.Directory, segments.front.name); + this.children ~= child; + } + // Recurse if needed + segments.popFront(); + return !segments.empty ? child.mkdir(NativePath(segments)) : child; + } + + /// + public bool isFile () const scope + { + return this.attributes.type == Type.File; + } + + /// + public bool isDirectory () const scope + { + return this.attributes.type == Type.Directory; + } + + /// Reads a file, returns the content as `ubyte[]` + public ubyte[] readFile (in NativePath path) const scope + { + auto entry = this.lookup(path); + enforce(entry !is null, "No such file: " ~ path.toNativeString()); + enforce(entry.attributes.type == Type.File, "Trying to read a directory"); + // This is a hack to make poisoning a file possible. + // However, it is rather crude and doesn't allow to poison directory. + // Consider introducing a derived type to allow it. + assert(entry.content != "poison".representation, + "Trying to access poisoned path: " ~ path.toNativeString()); + return entry.content.dup; + } + + /// Reads a file, returns the content as text + public string readText (in NativePath path) const scope + { + import std.utf : validate; + + const content = this.readFile(path); + // Ignore BOM: If it's needed for a test, add support for it. + validate(cast(const(char[])) content); + // `readFile` just `dup` the content, so it's safe to cast. + return cast(string) content; + } + + /// Ditto + public void writeFile (in NativePath path, const(ubyte)[] data) scope + { + enforce(!path.endsWithSlash(), + "Cannot write to directory: " ~ path.toNativeString()); + if (auto file = this.lookup(path)) { + // If the file already exists, override it + enforce(file.attributes.type == Type.File, + "Trying to write to directory: " ~ path.toNativeString()); + file.content = data.dup; + } else { + auto p = this.getParent(path); + auto file = new FSEntry(p, Type.File, path.head.name()); + file.content = data.dup; + p.children ~= file; + } + } + + /** Remove a file + * + * Always error if the target is a directory. + * Does not error if the target does not exists + * and `force` is set to `true`. + * + * Params: + * path = Path to the file to remove + * force = Whether to ignore non-existing file, + * default to `false`. + */ + public void removeFile (in NativePath path, bool force = false) + { + import std.algorithm.searching : countUntil; + + assert(!path.empty, "Empty path provided to `removeFile`"); + enforce(!path.endsWithSlash(), + "Cannot remove file with directory path: " ~ path.toNativeString()); + auto p = this.getParent(path, force); + const idx = p.children.countUntil!(e => e.name == path.head.name()); + if (idx < 0) { + enforce(force, + "removeFile: No such file: " ~ path.toNativeString()); + } else { + enforce(p.children[idx].attributes.type == Type.File, + "removeFile called on a directory: " ~ path.toNativeString()); + p.children = p.children[0 .. idx] ~ p.children[idx + 1 .. $]; + } + } + + /** Remove a directory + * + * Remove an existing empty directory. + * If `force` is set to `true`, no error will be thrown + * if the directory is empty or non-existing. + * + * Params: + * path = Path to the directory to remove + * force = Whether to ignore non-existing / non-empty directories, + * default to `false`. + */ + public void removeDir (in NativePath path, bool force = false) + { + import std.algorithm.searching : countUntil; + + assert(!path.empty, "Empty path provided to `removeFile`"); + auto p = this.getParent(path, force); + const idx = p.children.countUntil!(e => e.name == path.head.name()); + if (idx < 0) { + enforce(force, + "removeDir: No such directory: " ~ path.toNativeString()); + } else { + enforce(p.children[idx].attributes.type == Type.Directory, + "removeDir called on a file: " ~ path.toNativeString()); + enforce(force || p.children[idx].children.length == 0, + "removeDir called on non-empty directory: " ~ path.toNativeString()); + p.children = p.children[0 .. idx] ~ p.children[idx + 1 .. $]; + } + } + + /// Implement `std.file.setTimes` + public void setTimes (in SysTime accessTime, in SysTime modificationTime) + { + this.attributes.access = accessTime; + this.attributes.modification = modificationTime; + } + + /// Implement `std.file.setAttributes` + public void setAttributes (uint attributes) + { + this.attributes.attrs = attributes; + } +} diff --git a/source/dub/internal/io/realfs.d b/source/dub/internal/io/realfs.d new file mode 100644 index 0000000..9497169 --- /dev/null +++ b/source/dub/internal/io/realfs.d @@ -0,0 +1,102 @@ +/******************************************************************************* + + An implementation of `Filesystem` using vibe.d functions + +*******************************************************************************/ + +module dub.internal.io.realfs; + +public import dub.internal.io.filesystem; + +/// Ditto +public final class RealFS : Filesystem { + static import dub.internal.vibecompat.core.file; + static import std.file; + + /// + private NativePath path_; + + /// + public this (NativePath cwd = NativePath(std.file.getcwd())) + scope @safe pure nothrow @nogc + { + this.path_ = cwd; + } + + public override NativePath getcwd () const scope + { + return this.path_; + } + + /// + protected override bool existsDirectory (in NativePath path) const scope + { + return dub.internal.vibecompat.core.file.existsDirectory(path); + } + + /// Ditto + protected override void mkdir (in NativePath path) scope + { + dub.internal.vibecompat.core.file.ensureDirectory(path); + } + + /// Ditto + protected override bool existsFile (in NativePath path) const scope + { + return dub.internal.vibecompat.core.file.existsFile(path); + } + + /// Ditto + protected override void writeFile (in NativePath path, const(ubyte)[] data) + scope + { + return dub.internal.vibecompat.core.file.writeFile(path, data); + } + + /// Reads a file, returns the content as `ubyte[]` + public override ubyte[] readFile (in NativePath path) const scope + { + return cast(ubyte[]) std.file.read(path.toNativeString()); + } + + /// Ditto + protected override string readText (in NativePath path) const scope + { + return dub.internal.vibecompat.core.file.readText(path); + } + + /// Ditto + protected override IterateDirDg iterateDirectory (in NativePath path) scope + { + return dub.internal.vibecompat.core.file.iterateDirectory(path); + } + + /// Ditto + protected override void removeFile (in NativePath path, bool force = false) scope + { + return std.file.remove(path.toNativeString()); + } + + /// + public override void removeDir (in NativePath path, bool force = false) + { + if (force) + std.file.rmdirRecurse(path.toNativeString()); + else + std.file.rmdir(path.toNativeString()); + } + + /// Ditto + protected override void setTimes (in NativePath path, in SysTime accessTime, + in SysTime modificationTime) + { + std.file.setTimes( + path.toNativeString(), accessTime, modificationTime); + } + + /// Ditto + protected override void setAttributes (in NativePath path, uint attributes) + { + std.file.setAttributes(path.toNativeString(), attributes); + } +} diff --git a/source/dub/internal/utils.d b/source/dub/internal/utils.d index 2faba88..b8b174c 100644 --- a/source/dub/internal/utils.d +++ b/source/dub/internal/utils.d @@ -21,6 +21,7 @@ import std.exception : enforce; import std.file; import std.format; +import std.range; import std.string : format; import std.process; import std.traits : isIntegral; @@ -145,23 +146,11 @@ moveFile(tmppath, path); } -deprecated("specify a working directory explicitly") -void runCommand(string command, string[string] env = null) -{ - runCommands((&command)[0 .. 1], env, null); -} - void runCommand(string command, string[string] env, string workDir) { runCommands((&command)[0 .. 1], env, workDir); } -deprecated("specify a working directory explicitly") -void runCommands(in string[] commands, string[string] env = null) -{ - runCommands(commands, env, null); -} - void runCommands(in string[] commands, string[string] env, string workDir) { import std.stdio : stdin, stdout, stderr, File; @@ -797,3 +786,41 @@ file, line); } } + +/** Filters a forward range with the given predicate and returns a prefix range. + + This function filters elements in-place, as opposed to returning a new + range. This can be particularly useful when working with arrays, as this + does not require any memory allocations. + + This function guarantees that `pred` is called exactly once per element and + deterministically destroys any elements for which `pred` returns `false`. +*/ +auto filterInPlace(alias pred, R)(R elems) + if (isForwardRange!R) +{ + import std.algorithm.mutation : move; + + R telems = elems.save; + bool any_removed = false; + size_t nret = 0; + foreach (ref el; elems.save) { + if (pred(el)) { + if (any_removed) move(el, telems.front); + telems.popFront(); + nret++; + } else any_removed = true; + } + return elems.takeExactly(nret); +} + +/// +unittest { + int[] arr = [1, 2, 3, 4, 5, 6, 7, 8]; + + arr = arr.filterInPlace!(e => e % 2 == 0); + assert(arr == [2, 4, 6, 8]); + + arr = arr.filterInPlace!(e => e < 5); + assert(arr == [2, 4]); +} diff --git a/source/dub/packagemanager.d b/source/dub/packagemanager.d index eb17a9f..79029cc 100644 --- a/source/dub/packagemanager.d +++ b/source/dub/packagemanager.d @@ -8,8 +8,8 @@ module dub.packagemanager; import dub.dependency; +import dub.internal.io.filesystem; import dub.internal.utils; -import dub.internal.vibecompat.core.file : FileInfo; import dub.internal.vibecompat.data.json; import dub.internal.vibecompat.inet.path; import dub.internal.logging; @@ -30,6 +30,7 @@ import std.exception; import std.range; import std.string; +import std.typecons; import std.zip; @@ -59,6 +60,15 @@ } } +/// A SelectionsFile associated with its file-system path. +struct SelectionsFileLookupResult { + /// The absolute path to the dub.selections.json file + /// (potentially inherited from a parent directory of the root package). + NativePath absolutePath; + /// The parsed dub.selections.json file. + SelectionsFile selectionsFile; +} + /// The PackageManager can retrieve present packages and get / remove /// packages. class PackageManager { @@ -85,17 +95,29 @@ */ Location[] m_repositories; /** - * Whether `refresh` has been called or not + * Whether `refreshLocal` / `refreshCache` has been called or not * - * Dub versions because v1.31 eagerly scan all available repositories, - * leading to slowdown for the most common operation - `dub build` with - * already resolved dependencies. - * From v1.31 onward, those locations are not scanned eagerly, - * unless one of the function requiring eager scanning does, - * such as `getBestPackage` - as it needs to iterate the list - * of available packages. + * User local cache can get pretty large, and we want to avoid our build + * time being dependent on their size. However, in order to support + * local packages and overrides, we were scanning the whole cache prior + * to v1.39.0 (although attempts at fixing this behavior were made + * earlier). Those booleans record whether we have been semi-initialized + * (local packages and overrides have been loaded) or fully initialized + * (all caches have been scanned), the later still being required for + * some API (e.g. `getBestPackage` or `getPackageIterator`). */ - bool m_initialized; + enum InitializationState { + /// No `refresh*` function has been called + none, + /// `refreshLocal` has been called + partial, + /// `refreshCache` (and `refreshLocal`) has been called + full, + } + /// Ditto + InitializationState m_state; + /// The `Filesystem` object, used to interact with directory / files + Filesystem fs; } /** @@ -110,20 +132,40 @@ */ this(NativePath path) { + import dub.internal.io.realfs; + this.fs = new RealFS(); this.m_internal.searchPath = [ path ]; this.refresh(); } this(NativePath package_path, NativePath user_path, NativePath system_path, bool refresh_packages = true) { - m_repositories = [ - Location(package_path ~ ".dub/packages/"), - Location(user_path ~ "packages/"), - Location(system_path ~ "packages/")]; - + import dub.internal.io.realfs; + this(new RealFS(), package_path ~ ".dub/packages/", + user_path ~ "packages/", system_path ~ "packages/"); if (refresh_packages) refresh(); } + /** + * Instantiate a `PackageManager` with the provided `Filesystem` and paths + * + * Unlike the other overload, paths are taken as-if, e.g. `packages/` is not + * appended to them. + * + * Params: + * fs = Filesystem abstraction to handle all folder/file I/O. + * local = Path to the local package cache (usually the one in the project), + * whih takes preference over `user` and `system`. + * user = Path to the user package cache (usually ~/.dub/packages/), takes + * precedence over `system` but not over `local`. + * system = Path to the system package cache, this has the least precedence. + */ + public this(Filesystem fs, NativePath local, NativePath user, NativePath system) + { + this.fs = fs; + this.m_repositories = [ Location(local), Location(user), Location(system) ]; + } + /** Gets/sets the list of paths to search for local packages. */ @property void searchPath(NativePath[] paths) @@ -190,8 +232,10 @@ * A `Package` if one was found, `null` if none exists. */ protected Package lookup (in PackageName name, in Version vers) { - if (!this.m_initialized) - this.refresh(); + // This is the only place we can get away with lazy initialization, + // since we know exactly what package and version we want. + // However, it is also the most often called API. + this.ensureInitialized(InitializationState.partial); if (auto pkg = this.m_internal.lookup(name, vers)) return pkg; @@ -390,7 +434,7 @@ const PackageName pname = parent ? PackageName(parent.name) : PackageName.init; - string text = this.readText(recipe); + string text = this.fs.readText(recipe); auto content = parsePackageRecipe( text, recipe.toNativeString(), pname, null, mode); auto ret = new Package(content, path, parent, version_); @@ -411,7 +455,7 @@ { foreach (file; packageInfoFiles) { auto filename = directory ~ file.filename; - if (this.existsFile(filename)) return filename; + if (this.fs.existsFile(filename)) return filename; } return NativePath.init; } @@ -464,13 +508,19 @@ string gitReference = repo.ref_.chompPrefix("~"); NativePath destination = this.getPackagePath(PlacementLocation.user, name, repo.ref_); - foreach (p; getPackageIterator(name.toString())) { - if (p.path == destination) { - return p; - } - } - - if (!this.gitClone(repo.remote, gitReference, destination)) + // Before doing a git clone, let's see if the package exists locally + if (this.fs.existsDirectory(destination)) { + // It exists, check if we already loaded it. + // Either we loaded it on refresh and it's in PlacementLocation.user, + // or we just added it and it's in m_internal. + foreach (p; this.m_internal.fromPath) + if (p.path == destination) + return p; + if (this.m_repositories.length) + foreach (p; this.m_repositories[PlacementLocation.user].fromPath) + if (p.path == destination) + return p; + } else if (!this.gitClone(repo.remote, gitReference, destination)) return null; Package result = this.load(destination); @@ -628,9 +678,8 @@ */ int delegate(int delegate(ref Package)) getPackageIterator() { - // See `m_initialized` documentation - if (!this.m_initialized) - this.refresh(); + // This API requires full knowledge of the package cache + this.ensureInitialized(InitializationState.full); int iterator(int delegate(ref Package) del) { @@ -806,17 +855,15 @@ Package store(ubyte[] data, PlacementLocation dest, in PackageName name, in Version vers) { - import dub.internal.vibecompat.core.file; - assert(!name.sub.length, "Cannot store a subpackage, use main package instead"); NativePath dstpath = this.getPackagePath(dest, name, vers.toString()); - this.ensureDirectory(dstpath.parentPath()); + this.fs.mkdir(dstpath.parentPath()); const lockPath = dstpath.parentPath() ~ ".lock"; // possibly wait for other dub instance import core.time : seconds; auto lock = lockFile(lockPath.toNativeString(), 30.seconds); - if (this.existsFile(dstpath)) { + if (this.fs.existsFile(dstpath)) { return this.getPackage(name, vers, dest); } return this.store_(data, dstpath, name, vers); @@ -824,16 +871,16 @@ /// Backward-compatibility for deprecated overload, simplify once `storeFetchedPatch` /// is removed - private Package store_(ubyte[] data, NativePath destination, + protected Package store_(ubyte[] data, NativePath destination, in PackageName name, in Version vers) { - import dub.internal.vibecompat.core.file; + import dub.recipe.json : toJson; import std.range : walkLength; logDebug("Placing package '%s' version '%s' to location '%s'", name, vers, destination.toNativeString()); - enforce(!this.existsFile(destination), + enforce(!this.fs.existsFile(destination), "%s (%s) needs to be removed from '%s' prior placement." .format(name, vers, destination)); @@ -866,13 +913,13 @@ import std.datetime : DosFileTimeToSysTime; auto mtime = DosFileTimeToSysTime(am.time); - this.setTimes(path, mtime, mtime); + this.fs.setTimes(path, mtime, mtime); if (auto attrs = am.fileAttributes) - this.setAttributes(path, attrs); + this.fs.setAttributes(path, attrs); } // extract & place - this.ensureDirectory(destination); + this.fs.mkdir(destination); logDebug("Copying all files..."); int countFiles = 0; foreach(ArchiveMember a; archive.directory) { @@ -882,9 +929,9 @@ logDebug("Creating %s", cleanedPath); if (dst_path.endsWithSlash) { - this.ensureDirectory(dst_path); + this.fs.mkdir(dst_path); } else { - this.ensureDirectory(dst_path.parentPath); + this.fs.mkdir(dst_path.parentPath); // for symlinks on posix systems, use the symlink function to // create them. Windows default unzip doesn't handle symlinks, // so we don't need to worry about it for Windows. @@ -900,7 +947,7 @@ } } - this.writeFile(dst_path, archive.expand(a)); + this.fs.writeFile(dst_path, archive.expand(a)); setAttributes(dst_path, a); symlink_exit: ++countFiles; @@ -913,8 +960,10 @@ if (pack.recipePath.head != defaultPackageFilename) // Storeinfo saved a default file, this could be different to the file from the zip. - this.removeFile(pack.recipePath); - pack.storeInfo(); + this.fs.removeFile(pack.recipePath); + auto app = appender!string(); + app.writePrettyJsonString(pack.recipe.toJson()); + this.fs.writeFile(pack.recipePath.parentPath ~ defaultPackageFilename, app.data); addPackages(this.m_internal.localPackages, pack); return pack; } @@ -970,8 +1019,7 @@ // As we iterate over `localPackages` we need it to be populated // In theory we could just populate that specific repository, // but multiple calls would then become inefficient. - if (!this.m_initialized) - this.refresh(); + this.ensureInitialized(InitializationState.full); path.endsWithSlash = true; auto pack = this.load(path); @@ -1002,8 +1050,7 @@ // As we iterate over `localPackages` we need it to be populated // In theory we could just populate that specific repository, // but multiple calls would then become inefficient. - if (!this.m_initialized) - this.refresh(); + this.ensureInitialized(InitializationState.full); path.endsWithSlash = true; Package[]* packs = &m_repositories[type].localPackages; @@ -1046,29 +1093,52 @@ { if (refresh) logDiagnostic("Refreshing local packages (refresh existing: true)..."); - this.refresh_(refresh); + else + logDiagnostic("Scanning local packages..."); + + this.refreshLocal(refresh); + this.refreshCache(refresh); } void refresh() { - this.refresh_(false); + logDiagnostic("Scanning local packages..."); + this.refreshLocal(false); + this.refreshCache(false); } - private void refresh_(bool refresh) + /// Private API to ensure a level of initialization + private void ensureInitialized(InitializationState state) { - if (!refresh) - logDiagnostic("Scanning local packages..."); + if (this.m_state >= state) + return; + if (state == InitializationState.partial) + this.refreshLocal(false); + else + this.refresh(); + } + /// Refresh pay-as-you-go: Only load local packages, not the full cache + private void refreshLocal(bool refresh) { foreach (ref repository; this.m_repositories) repository.scanLocalPackages(refresh, this); - this.m_internal.scan(this, refresh); + foreach (ref repository; this.m_repositories) { + auto existing = refresh ? null : repository.fromPath; + foreach (path; repository.searchPath) + repository.scanPackageFolder(path, this, existing); + repository.loadOverrides(this); + } + if (this.m_state < InitializationState.partial) + this.m_state = InitializationState.partial; + } + + /// Refresh the full cache, a potentially expensive operation + private void refreshCache(bool refresh) + { foreach (ref repository; this.m_repositories) repository.scan(this, refresh); - - foreach (ref repository; this.m_repositories) - repository.loadOverrides(this); - this.m_initialized = true; + this.m_state = InitializationState.full; } alias Hash = ubyte[]; @@ -1106,6 +1176,59 @@ } /** + * Loads the selections file (`dub.selections.json`) + * + * The selections file is only used for the root package / project. + * However, due to it being a filesystem interaction, it is managed + * from the `PackageManager`. + * + * Params: + * absProjectPath = The absolute path to the root package/project for + * which to load the selections file. + * + * Returns: + * Either `null` (if no selections file exists or parsing encountered an error), + * or a `SelectionsFileLookupResult`. Note that the nested `SelectionsFile` + * might use an unsupported version (see `SelectionsFile` documentation). + */ + Nullable!SelectionsFileLookupResult readSelections(in NativePath absProjectPath) + in (absProjectPath.absolute) { + import dub.internal.configy.Read; + + alias N = typeof(return); + + // check for dub.selections.json in root project dir first, then walk up its + // parent directories and look for inheritable dub.selections.json files + const path = this.findSelections(absProjectPath); + if (path.empty) return N.init; + const content = this.fs.readText(path); + // TODO: Remove `StrictMode.Warn` after v1.40 release + // The default is to error, but as the previous parser wasn't + // complaining, we should first warn the user. + auto selections = wrapException(parseConfigString!SelectionsFile( + content, path.toNativeString(), StrictMode.Warn)); + // Could not parse file + if (selections.isNull()) + return N.init; + // Non-inheritable selections found + if (!path.startsWith(absProjectPath) && !selections.get().inheritable) + return N.init; + return N(SelectionsFileLookupResult(path, selections.get())); + } + + /// Helper function to walk up the filesystem and find `dub.selections.json` + private NativePath findSelections(in NativePath dir) + { + const path = dir ~ "dub.selections.json"; + if (this.fs.existsFile(path)) + return path; + if (!dir.hasParentPath) + return NativePath.init; + return this.findSelections(dir.parentPath); + + } + + /** * Writes the selections file (`dub.selections.json`) * * The selections file is only used for the root package / project. @@ -1122,9 +1245,9 @@ bool overwrite = true) { const path = project.path ~ "dub.selections.json"; - if (!overwrite && this.existsFile(path)) + if (!overwrite && this.fs.existsFile(path)) return; - this.writeFile(path, selectionsToString(selections)); + this.fs.writeFile(path, selectionsToString(selections)); } /// Package function to avoid code duplication with deprecated @@ -1134,12 +1257,14 @@ { Json json = selectionsToJSON(s); assert(json.type == Json.Type.object); - assert(json.length == 2); + assert(json.length == 2 || json.length == 3); assert(json["versions"].type != Json.Type.undefined); auto result = appender!string(); result.put("{\n\t\"fileVersion\": "); result.writeJsonString(json["fileVersion"]); + if (s.inheritable) + result.put(",\n\t\"inheritable\": true"); result.put(",\n\t\"versions\": {"); auto vers = json["versions"].get!(Json[string]); bool first = true; @@ -1160,6 +1285,8 @@ { Json serialized = Json.emptyObject; serialized["fileVersion"] = s.fileVersion; + if (s.inheritable) + serialized["inheritable"] = true; serialized["versions"] = Json.emptyObject; foreach (p, dep; s.versions) serialized["versions"][p] = dep.toJson(true); @@ -1196,81 +1323,6 @@ } } } - - /// Used for dependency injection - protected bool existsDirectory(NativePath path) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.existsDirectory(path); - } - - /// Ditto - protected void ensureDirectory(NativePath path) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.ensureDirectory(path); - } - - /// Ditto - protected bool existsFile(NativePath path) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.existsFile(path); - } - - /// Ditto - protected void writeFile(NativePath path, const(ubyte)[] data) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.writeFile(path, data); - } - - /// Ditto - protected void writeFile(NativePath path, const(char)[] data) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.writeFile(path, data); - } - - /// Ditto - protected string readText(NativePath path) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.readText(path); - } - - /// Ditto - protected alias IterateDirDg = int delegate(scope int delegate(ref FileInfo)); - - /// Ditto - protected IterateDirDg iterateDirectory(NativePath path) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.iterateDirectory(path); - } - - /// Ditto - protected void removeFile(NativePath path) - { - static import dub.internal.vibecompat.core.file; - return dub.internal.vibecompat.core.file.removeFile(path); - } - - /// Ditto - protected void setTimes(in NativePath path, in SysTime accessTime, - in SysTime modificationTime) - { - static import std.file; - std.file.setTimes( - path.toNativeString(), accessTime, modificationTime); - } - - /// Ditto - protected void setAttributes(in NativePath path, uint attributes) - { - static import std.file; - std.file.setAttributes(path.toNativeString(), attributes); - } } deprecated(OverrideDepMsg) @@ -1414,11 +1466,11 @@ { this.overrides = null; auto ovrfilepath = this.packagePath ~ LocalOverridesFilename; - if (mgr.existsFile(ovrfilepath)) { + if (mgr.fs.existsFile(ovrfilepath)) { logWarn("Found local override file: %s", ovrfilepath); logWarn(OverrideDepMsg); logWarn("Replace with a path-based dependency in your project or a custom cache path"); - const text = mgr.readText(ovrfilepath); + const text = mgr.fs.readText(ovrfilepath); auto json = parseJsonString(text, ovrfilepath.toNativeString()); foreach (entry; json) { PackageOverride_ ovr; @@ -1445,10 +1497,10 @@ newlist ~= jovr; } auto path = this.packagePath; - mgr.ensureDirectory(path); + mgr.fs.mkdir(path); auto app = appender!string(); app.writePrettyJsonString(Json(newlist)); - mgr.writeFile(path ~ LocalOverridesFilename, app.data); + mgr.fs.writeFile(path ~ LocalOverridesFilename, app.data); } private void writeLocalPackageList(PackageManager mgr) @@ -1471,10 +1523,10 @@ } NativePath path = this.packagePath; - mgr.ensureDirectory(path); + mgr.fs.mkdir(path); auto app = appender!string(); app.writePrettyJsonString(Json(newlist)); - mgr.writeFile(path ~ LocalPackagesFilename, app.data); + mgr.fs.writeFile(path ~ LocalPackagesFilename, app.data); } // load locally defined packages @@ -1485,10 +1537,10 @@ NativePath[] paths; try { auto local_package_file = list_path ~ LocalPackagesFilename; - if (!manager.existsFile(local_package_file)) return; + if (!manager.fs.existsFile(local_package_file)) return; logDiagnostic("Loading local package map at %s", local_package_file.toNativeString()); - const text = manager.readText(local_package_file); + const text = manager.fs.readText(local_package_file); auto packlist = parseJsonString( text, local_package_file.toNativeString()); enforce(packlist.type == Json.Type.array, LocalPackagesFilename ~ " must contain an array."); @@ -1563,7 +1615,7 @@ void scanPackageFolder(NativePath path, PackageManager mgr, Package[] existing_packages) { - if (!mgr.existsDirectory(path)) + if (!mgr.fs.existsDirectory(path)) return; void loadInternal (NativePath pack_path, NativePath packageFile) @@ -1587,7 +1639,7 @@ } logDebug("iterating dir %s", path.toNativeString()); - try foreach (pdir; mgr.iterateDirectory(path)) { + try foreach (pdir; mgr.fs.iterateDirectory(path)) { logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name); if (!pdir.isDirectory) continue; @@ -1609,10 +1661,10 @@ // This is the most common code path // Iterate over versions of a package - foreach (versdir; mgr.iterateDirectory(pack_path)) { + foreach (versdir; mgr.fs.iterateDirectory(pack_path)) { if (!versdir.isDirectory) continue; auto vers_path = pack_path ~ versdir.name ~ (pdir.name ~ "/"); - if (!mgr.existsDirectory(vers_path)) continue; + if (!mgr.fs.existsDirectory(vers_path)) continue; packageFile = mgr.findPackageFile(vers_path); loadInternal(vers_path, packageFile); } @@ -1682,7 +1734,7 @@ string versStr = vers.toString(); const path = this.getPackagePath(name, versStr); - if (!mgr.existsDirectory(path)) + if (!mgr.fs.existsDirectory(path)) return null; logDiagnostic("Lazily loading package %s:%s from %s", name.main, vers, path); diff --git a/source/dub/project.d b/source/dub/project.d index 50d27a3..575e8eb 100644 --- a/source/dub/project.d +++ b/source/dub/project.d @@ -43,6 +43,7 @@ PackageManager m_packageManager; Package m_rootPackage; Package[] m_dependencies; + Package[string] m_dependenciesByName; Package[][Package] m_dependees; SelectedVersions m_selections; string[] m_missingDependencies; @@ -75,7 +76,7 @@ /// Ditto this(PackageManager package_manager, Package pack) { - auto selections = Project.loadSelections(pack); + auto selections = Project.loadSelections(pack.path, package_manager); this(package_manager, pack, selections); } @@ -97,41 +98,34 @@ * is returned. * * Params: - * pack = Package to load the selection file from. + * packPath = Absolute path of the Package to load the selection file from. * * Returns: * Always a non-null instance. */ - static package SelectedVersions loadSelections(in Package pack) + static package SelectedVersions loadSelections(in NativePath packPath, PackageManager mgr) { import dub.version_; import dub.internal.dyaml.stdsumtype; - auto selverfile = (pack.path ~ SelectedVersions.defaultFile).toNativeString(); - - // No file exists - if (!existsFile(selverfile)) + auto lookupResult = mgr.readSelections(packPath); + if (lookupResult.isNull()) // no file, or parsing error (displayed to the user) return new SelectedVersions(); - // TODO: Remove `StrictMode.Warn` after v1.40 release - // The default is to error, but as the previous parser wasn't - // complaining, we should first warn the user. - auto selected = parseConfigFileSimple!SelectionsFile(selverfile, StrictMode.Warn); - - // Parsing error, it will be displayed to the user - if (selected.isNull()) - return new SelectedVersions(); - - return selected.get().content.match!( + auto r = lookupResult.get(); + return r.selectionsFile.content.match!( (Selections!0 s) { logWarnTag("Unsupported version", "File %s has fileVersion %s, which is not yet supported by DUB %s.", - selverfile, s.fileVersion, dubVersion); + r.absolutePath, s.fileVersion, dubVersion); logWarn("Ignoring selections file. Use a newer DUB version " ~ "and set the appropriate toolchainRequirements in your recipe file"); return new SelectedVersions(); }, - (Selections!1 s) => new SelectedVersions(s), + (Selections!1 s) { + auto selectionsDir = r.absolutePath.parentPath; + return new SelectedVersions(s, selectionsDir.relativeTo(packPath)); + }, ); } @@ -227,9 +221,8 @@ */ inout(Package) getDependency(string name, bool is_optional) inout { - foreach(dp; m_dependencies) - if( dp.name == name ) - return dp; + if (auto pp = name in m_dependenciesByName) + return *pp; if (!is_optional) throw new Exception("Unknown dependency: "~name); else return null; } @@ -519,8 +512,10 @@ void reinit() { m_dependencies = null; + m_dependenciesByName = null; m_missingDependencies = []; collectDependenciesRec(m_rootPackage); + foreach (p; m_dependencies) m_dependenciesByName[p.name] = p; m_missingDependencies.sort(); } @@ -643,150 +638,173 @@ /// Returns a map with the configuration for all packages in the dependency tree. string[string] getPackageConfigs(in BuildPlatform platform, string config, bool allow_non_library = true) const { - struct Vertex { string pack, config; } - struct Edge { size_t from, to; } + import std.typecons : Rebindable, rebindable; + import std.range : only; + // prepare by collecting information about all packages in the project + // qualified names and dependencies are cached, to avoid recomputing + // them multiple times during the algorithm + auto packages = collectPackageInformation(); + + // graph of the project's package configuration dependencies + // (package, config) -> (sub-package, sub-config) + static struct Vertex { size_t pack = size_t.max; string config; } + static struct Edge { size_t from, to; } Vertex[] configs; + void[0][Vertex] configs_set; Edge[] edges; - string[][string] parents; - parents[m_rootPackage.name] = null; - foreach (p; getTopologicalPackageList()) - foreach (d; p.getAllDependencies()) - parents[d.name.toString()] ~= p.name; - size_t createConfig(string pack, string config) { + + size_t createConfig(size_t pack_idx, string config) { foreach (i, v; configs) - if (v.pack == pack && v.config == config) + if (v.pack == pack_idx && v.config == config) return i; - assert(pack !in m_overriddenConfigs || config == m_overriddenConfigs[pack]); - logDebug("Add config %s %s", pack, config); - configs ~= Vertex(pack, config); + + auto pname = packages[pack_idx].name; + assert(pname !in m_overriddenConfigs || config == m_overriddenConfigs[pname]); + logDebug("Add config %s %s", pname, config); + auto cfg = Vertex(pack_idx, config); + configs ~= cfg; + configs_set[cfg] = (void[0]).init; return configs.length-1; } - bool haveConfig(string pack, string config) { - return configs.any!(c => c.pack == pack && c.config == config); + bool haveConfig(size_t pack_idx, string config) { + return (Vertex(pack_idx, config) in configs_set) !is null; } - size_t createEdge(size_t from, size_t to) { - auto idx = edges.countUntil(Edge(from, to)); - if (idx >= 0) return idx; - logDebug("Including %s %s -> %s %s", configs[from].pack, configs[from].config, configs[to].pack, configs[to].config); - edges ~= Edge(from, to); - return edges.length-1; - } - - void removeConfig(size_t i) { - logDebug("Eliminating config %s for %s", configs[i].config, configs[i].pack); + void removeConfig(size_t config_index) { + logDebug("Eliminating config %s for %s", configs[config_index].config, configs[config_index].pack); auto had_dep_to_pack = new bool[configs.length]; auto still_has_dep_to_pack = new bool[configs.length]; - edges = edges.filter!((e) { - if (e.to == i) { - had_dep_to_pack[e.from] = true; - return false; - } else if (configs[e.to].pack == configs[i].pack) { - still_has_dep_to_pack[e.from] = true; - } - if (e.from == i) return false; - return true; - }).array; + // eliminate all edges that connect to config 'config_index' and + // track all connected configs + edges = edges.filterInPlace!((e) { + if (e.to == config_index) { + had_dep_to_pack[e.from] = true; + return false; + } else if (configs[e.to].pack == configs[config_index].pack) { + still_has_dep_to_pack[e.from] = true; + } - configs[i] = Vertex.init; // mark config as removed + return e.from != config_index; + }); + + // mark config as removed + configs_set.remove(configs[config_index]); + configs[config_index] = Vertex.init; // also remove any configs that cannot be satisfied anymore foreach (j; 0 .. configs.length) - if (j != i && had_dep_to_pack[j] && !still_has_dep_to_pack[j]) + if (j != config_index && had_dep_to_pack[j] && !still_has_dep_to_pack[j]) removeConfig(j); } - bool isReachable(string pack, string conf) { - if (pack == configs[0].pack && configs[0].config == conf) return true; - foreach (e; edges) - if (configs[e.to].pack == pack && configs[e.to].config == conf) - return true; - return false; - //return (pack == configs[0].pack && conf == configs[0].config) || edges.canFind!(e => configs[e.to].pack == pack && configs[e.to].config == config); - } - + bool[] reachable = new bool[packages.length]; // reused to avoid continuous re-allocation bool isReachableByAllParentPacks(size_t cidx) { - bool[string] r; - foreach (p; parents[configs[cidx].pack]) r[p] = false; + foreach (p; packages[configs[cidx].pack].parents) reachable[p] = false; foreach (e; edges) { if (e.to != cidx) continue; - if (auto pp = configs[e.from].pack in r) *pp = true; + reachable[configs[e.from].pack] = true; } - foreach (bool v; r) if (!v) return false; + foreach (p; packages[configs[cidx].pack].parents) + if (!reachable[p]) + return false; return true; } - string[] allconfigs_path; - - void determineDependencyConfigs(in Package p, string c) + string[][] depconfigs = new string[][](packages.length); + void determineDependencyConfigs(size_t pack_idx, string c) { + void[0][Edge] edges_set; + void createEdge(size_t from, size_t to) { + if (Edge(from, to) in edges_set) + return; + logDebug("Including %s %s -> %s %s", configs[from].pack, configs[from].config, configs[to].pack, configs[to].config); + edges ~= Edge(from, to); + edges_set[Edge(from, to)] = (void[0]).init; + } + + auto pack = &packages[pack_idx]; // below we call createConfig for the main package if // config.length is not zero. Carry on for that case, // otherwise we've handle the pair (p, c) already - if(haveConfig(p.name, c) && !(config.length && p.name == m_rootPackage.name && config == c)) + if(haveConfig(pack_idx, c) && !(config.length && pack.name == m_rootPackage.name && config == c)) return; - string[][string] depconfigs; - foreach (d; p.getAllDependencies()) { - auto dp = getDependency(d.name.toString(), true); - if (!dp) continue; + foreach (d; pack.dependencies) { + auto dp = packages.getPackageIndex(d.name.toString()); + if (dp == size_t.max) continue; - string[] cfgs; - if (auto pc = dp.name in m_overriddenConfigs) cfgs = [*pc]; - else { - auto subconf = p.getSubConfiguration(c, dp, platform); - if (!subconf.empty) cfgs = [subconf]; - else cfgs = dp.getPlatformConfigurations(platform); + depconfigs[dp].length = 0; + depconfigs[dp].assumeSafeAppend; + + void setConfigs(R)(R configs) { + configs + .filter!(c => haveConfig(dp, c)) + .each!((c) { depconfigs[dp] ~= c; }); } - cfgs = cfgs.filter!(c => haveConfig(d.name.toString(), c)).array; + if (auto pc = packages[dp].name in m_overriddenConfigs) { + setConfigs(only(*pc)); + } else { + auto subconf = pack.package_.getSubConfiguration(c, packages[dp].package_, platform); + if (!subconf.empty) setConfigs(only(subconf)); + else setConfigs(packages[dp].package_.getPlatformConfigurations(platform)); + } // if no valid configuration was found for a dependency, don't include the // current configuration - if (!cfgs.length) { - logDebug("Skip %s %s (missing configuration for %s)", p.name, c, dp.name); + if (!depconfigs[dp].length) { + logDebug("Skip %s %s (missing configuration for %s)", pack.name, c, packages[dp].name); return; } - depconfigs[d.name.toString()] = cfgs; } // add this configuration to the graph - size_t cidx = createConfig(p.name, c); - foreach (d; p.getAllDependencies()) - foreach (sc; depconfigs.get(d.name.toString(), null)) - createEdge(cidx, createConfig(d.name.toString(), sc)); + size_t cidx = createConfig(pack_idx, c); + foreach (d; pack.dependencies) { + if (auto pdp = d.name.toString() in packages) + foreach (sc; depconfigs[*pdp]) + createEdge(cidx, createConfig(*pdp, sc)); + } } - // create a graph of all possible package configurations (package, config) -> (sub-package, sub-config) - void determineAllConfigs(in Package p) + string[] allconfigs_path; + void determineAllConfigs(size_t pack_idx) { - auto idx = allconfigs_path.countUntil(p.name); - enforce(idx < 0, format("Detected dependency cycle: %s", (allconfigs_path[idx .. $] ~ p.name).join("->"))); - allconfigs_path ~= p.name; - scope (exit) allconfigs_path.length--; + auto pack = &packages[pack_idx]; - // first, add all dependency configurations - foreach (d; p.getAllDependencies) { - auto dp = getDependency(d.name.toString(), true); - if (!dp) continue; - determineAllConfigs(dp); + auto idx = allconfigs_path.countUntil(pack.name); + enforce(idx < 0, format("Detected dependency cycle: %s", (allconfigs_path[idx .. $] ~ pack.name).join("->"))); + allconfigs_path ~= pack.name; + scope (exit) { + allconfigs_path.length--; + allconfigs_path.assumeSafeAppend; } - // for each configuration, determine the configurations usable for the dependencies - if (auto pc = p.name in m_overriddenConfigs) - determineDependencyConfigs(p, *pc); - else - foreach (c; p.getPlatformConfigurations(platform, p is m_rootPackage && allow_non_library)) - determineDependencyConfigs(p, c); - } - if (config.length) createConfig(m_rootPackage.name, config); - determineAllConfigs(m_rootPackage); + // first, add all dependency configurations + foreach (d; pack.dependencies) + if (auto pi = d.name.toString() in packages) + determineAllConfigs(*pi); - // successively remove configurations until only one configuration per package is left + // for each configuration, determine the configurations usable for the dependencies + if (auto pc = pack.name in m_overriddenConfigs) + determineDependencyConfigs(pack_idx, *pc); + else + foreach (c; pack.package_.getPlatformConfigurations(platform, pack.package_ is m_rootPackage && allow_non_library)) + determineDependencyConfigs(pack_idx, c); + } + + + // first, create a graph of all possible package configurations + assert(packages[0].package_ is m_rootPackage); + if (config.length) createConfig(0, config); + determineAllConfigs(0); + + // then, successively remove configurations until only one configuration + // per package is left bool changed; do { // remove all configs that are not reachable by all parent packages @@ -794,7 +812,7 @@ foreach (i, ref c; configs) { if (c == Vertex.init) continue; // ignore deleted configurations if (!isReachableByAllParentPacks(i)) { - logDebug("%s %s NOT REACHABLE by all of (%s):", c.pack, c.config, parents[c.pack]); + logDebug("%s %s NOT REACHABLE by all of (%s):", c.pack, c.config, packages[c.pack].parents); removeConfig(i); changed = true; } @@ -802,10 +820,10 @@ // when all edges are cleaned up, pick one package and remove all but one config if (!changed) { - foreach (p; getTopologicalPackageList()) { + foreach (pidx; 0 .. packages.length) { size_t cnt = 0; foreach (i, ref c; configs) - if (c.pack == p.name && ++cnt > 1) { + if (c.pack == pidx && ++cnt > 1) { logDebug("NON-PRIMARY: %s %s", c.pack, c.config); removeConfig(i); } @@ -824,25 +842,74 @@ string[string] ret; foreach (c; configs) { if (c == Vertex.init) continue; // ignore deleted configurations - assert(ret.get(c.pack, c.config) == c.config, format("Conflicting configurations for %s found: %s vs. %s", c.pack, c.config, ret[c.pack])); - logDebug("Using configuration '%s' for %s", c.config, c.pack); - ret[c.pack] = c.config; + auto pname = packages[c.pack].name; + assert(ret.get(pname, c.config) == c.config, format("Conflicting configurations for %s found: %s vs. %s", pname, c.config, ret[pname])); + logDebug("Using configuration '%s' for %s", c.config, pname); + ret[pname] = c.config; } // check for conflicts (packages missing in the final configuration graph) - void checkPacksRec(in Package pack) { - auto pc = pack.name in ret; - enforce(pc !is null, "Could not resolve configuration for package "~pack.name); - foreach (p, dep; pack.getDependencies(*pc)) { + auto visited = new bool[](packages.length); + void checkPacksRec(size_t pack_idx) { + if (visited[pack_idx]) return; + visited[pack_idx] = true; + auto pname = packages[pack_idx].name; + auto pc = pname in ret; + enforce(pc !is null, "Could not resolve configuration for package "~pname); + foreach (p, dep; packages[pack_idx].package_.getDependencies(*pc)) { auto deppack = getDependency(p, dep.optional); - if (deppack) checkPacksRec(deppack); + if (deppack) checkPacksRec(packages[].countUntil!(p => p.package_ is deppack)); } } - checkPacksRec(m_rootPackage); + checkPacksRec(0); return ret; } + /** Returns an ordered list of all packages with the additional possibility + to look up by name. + */ + private auto collectPackageInformation() + const { + static struct PackageInfo { + const(Package) package_; + size_t[] parents; + string name; + PackageDependency[] dependencies; + } + + static struct PackageInfoAccessor { + private { + PackageInfo[] m_packages; + size_t[string] m_packageMap; + } + + private void initialize(P)(P all_packages, size_t reserve_count) + { + m_packages.reserve(reserve_count); + foreach (p; all_packages) { + auto pname = p.name; + m_packageMap[pname] = m_packages.length; + m_packages ~= PackageInfo(p, null, pname, p.getAllDependencies()); + } + foreach (pack_idx, ref pack_info; m_packages) + foreach (d; pack_info.dependencies) + if (auto pi = d.name.toString() in m_packageMap) + m_packages[*pi].parents ~= pack_idx; + } + + size_t length() const { return m_packages.length; } + const(PackageInfo)[] opIndex() const { return m_packages; } + ref const(PackageInfo) opIndex(size_t package_index) const { return m_packages[package_index]; } + size_t getPackageIndex(string package_name) const { return m_packageMap.get(package_name, size_t.max); } + const(size_t)* opBinaryRight(string op = "in")(string package_name) const { return package_name in m_packageMap; } + } + + PackageInfoAccessor ret; + ret.initialize(getTopologicalPackageList(), m_dependencies.length); + return ret; + } + /** * Fills `dst` with values from this project. * @@ -1809,6 +1876,23 @@ this.m_bare = false; } + /** Constructs a new non-empty version selection, prefixing relative path + selections with the specified prefix. + + To be used in cases where the "dub.selections.json" file isn't located + in the root package directory. + */ + public this(Selections!1 data, NativePath relPathPrefix) + { + this(data); + if (relPathPrefix.empty) return; + foreach (ref dep; m_selections.versions.byValue) { + const depPath = dep.path; + if (!depPath.empty && !depPath.absolute) + dep = Dependency(relPathPrefix ~ depPath); + } + } + /** Constructs a new version selection from JSON data. The structure of the JSON document must match the contents of the @@ -1853,6 +1937,7 @@ { m_selections.fileVersion = versions.m_selections.fileVersion; m_selections.versions = versions.m_selections.versions.dup; + m_selections.inheritable = versions.m_selections.inheritable; m_dirty = true; } @@ -2013,6 +2098,8 @@ clear(); m_selections.fileVersion = fileVersion; scope(failure) clear(); + if (auto p = "inheritable" in json) + m_selections.inheritable = p.get!bool; foreach (string p, dep; json["versions"]) m_selections.versions[p] = dependencyFromJson(dep); } diff --git a/source/dub/recipe/packagerecipe.d b/source/dub/recipe/packagerecipe.d index 44f34b0..da30eda 100644 --- a/source/dub/recipe/packagerecipe.d +++ b/source/dub/recipe/packagerecipe.d @@ -709,6 +709,8 @@ import std.array : join; if (dep == "no") return VersionRange.Invalid; + // `dmdLikeVersionToSemverLike` does not handle this, VersionRange does + if (dep == "*") return VersionRange.Any; return VersionRange.fromString(dep .splitter(' ') .map!(r => dmdLikeVersionToSemverLike(r)) diff --git a/source/dub/recipe/selection.d b/source/dub/recipe/selection.d index 11758e8..f77df43 100644 --- a/source/dub/recipe/selection.d +++ b/source/dub/recipe/selection.d @@ -45,6 +45,18 @@ } /** + * Whether this dub.selections.json can be inherited by nested projects + * without local dub.selections.json + */ + public bool inheritable () const @safe pure nothrow @nogc + { + return this.content.match!( + (const Selections!0 _) => false, + (const Selections!1 s) => s.inheritable, + ); + } + + /** * The content of this selections file * * The underlying content can be accessed using @@ -102,6 +114,10 @@ else static if (Version == 1) { /// The selected package and their matching versions public SelectedDependency[string] versions; + + /// Whether this dub.selections.json can be inherited by nested projects + /// without local dub.selections.json + @Optional public bool inheritable; } else static assert(false, "This version is not supported"); @@ -191,6 +207,7 @@ (Selections!1 s) => s, (s) { assert(0); return Selections!(1).init; }, ); + assert(!s.inheritable); assert(s.versions.length == 5); assert(s.versions["simple"] == Dependency(Version("1.5.6"))); assert(s.versions["branch"] == Dependency(Version("~master"))); @@ -199,6 +216,23 @@ assert(s.versions["repository"] == Dependency(Repository("git+https://github.com/dlang/dub", "123456123456123456"))); } +// with optional `inheritable` Boolean +unittest +{ + import dub.internal.configy.Read : parseConfigString; + + immutable string content = `{ + "fileVersion": 1, + "inheritable": true, + "versions": { + "simple": "1.5.6", + } +}`; + + auto s = parseConfigString!SelectionsFile(content, "/dev/null"); + assert(s.inheritable); +} + // Test reading an unsupported version unittest { diff --git a/source/dub/test/base.d b/source/dub/test/base.d index 89c0624..859ba2e 100644 --- a/source/dub/test/base.d +++ b/source/dub/test/base.d @@ -60,8 +60,9 @@ public import dub.dependency; public import dub.dub; public import dub.package_; +import dub.internal.io.mockfs; import dub.internal.vibecompat.core.file : FileInfo; -public import dub.internal.vibecompat.inet.path; +public import dub.internal.io.filesystem; import dub.packagemanager; import dub.packagesuppliers.packagesupplier; import dub.project; @@ -82,7 +83,7 @@ // which receives an `FSEntry` representing the root of the filesystem. // Various low-level functions are exposed (mkdir, writeFile, ...), // as well as higher-level functions (`writePackageFile`). - scope dub = new TestDub((scope FSEntry root) { + scope dub = new TestDub((scope Filesystem root) { // `a` will be loaded as the project while `b` will be loaded // as a simple package. The recipe files can be in JSON or SDL format, // here we use both to demonstrate this. @@ -127,6 +128,19 @@ /// Now actually upgrade dependencies in memory dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade); assert(dub.project.getDependency("b", true).version_ == Version("1.2.0")); + + /// Adding a package to the registry require the version and at list a recipe + dub.getRegistry().add(Version("1.3.0"), (scope Filesystem pkg) { + // This is required + pkg.writeFile(NativePath(`dub.sdl`), `name "b"`); + // Any other files can be present, as a normal package + pkg.mkdir(NativePath("source/b/")); + pkg.writeFile( + NativePath("main.d"), "module b.main; void main() {}"); + }); + // Fetch the package from the registry + dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade); + assert(dub.project.getDependency("b", true).version_ == Version("1.3.0")); } // TODO: Remove and handle logging the same way we handle other IO @@ -162,7 +176,17 @@ public class TestDub : Dub { /// The virtual filesystem that this instance acts on - public FSEntry fs; + public MockFS fs; + + /** + * Redundant reference to the registry + * + * We currently create 2 `MockPackageSupplier`s hidden behind a + * `FallbackPackageSupplier` (see base implementation). + * The fallback is never used, and we need to provide the user + * a mean to access the registry so they can add packages to it. + */ + protected MockPackageSupplier registry; /// Convenience constants for use in unittests version (Windows) @@ -198,13 +222,13 @@ ***************************************************************************/ - public this (scope void delegate(scope FSEntry root) dg = null, + public this (scope void delegate(scope Filesystem root) dg = null, string root = ProjectPath.toNativeString(), PackageSupplier[] extras = null, SkipPackageSuppliers skip = SkipPackageSuppliers.none) { /// Create the fs & its base structure - auto fs_ = new FSEntry(); + auto fs_ = new MockFS(); fs_.mkdir(Paths.temp); fs_.mkdir(Paths.systemSettings); fs_.mkdir(Paths.userSettings); @@ -222,12 +246,12 @@ PackageSupplier[] extras = null, SkipPackageSuppliers skip = SkipPackageSuppliers.none) { - alias TType = void delegate(scope FSEntry); + alias TType = void delegate(scope Filesystem); this(TType.init, root, extras, skip); } /// Internal constructor - private this(FSEntry fs_, string root, PackageSupplier[] extras, + private this(MockFS fs_, string root, PackageSupplier[] extras, SkipPackageSuppliers skip) { this.fs = fs_; @@ -253,7 +277,7 @@ ***************************************************************************/ - public TestDub newTest (scope void delegate(scope FSEntry root) dg = null, + public TestDub newTest (scope void delegate(scope Filesystem root) dg = null, string root = ProjectPath.toNativeString(), PackageSupplier[] extras = null, SkipPackageSuppliers skip = SkipPackageSuppliers.none) @@ -277,21 +301,14 @@ } /// See `MockPackageSupplier` documentation for this class' implementation - protected override PackageSupplier makePackageSupplier(string url) const + protected override PackageSupplier makePackageSupplier(string url) { - return new MockPackageSupplier(url); + auto r = new MockPackageSupplier(url); + if (this.registry is null) + this.registry = r; + return r; } - /// Loads a specific package as the main project package (can be a sub package) - public override void loadPackage(Package pack) - { - auto selections = this.packageManager.loadSelections(pack); - m_project = new Project(m_packageManager, pack, selections); - } - - /// Reintroduce parent overloads - public alias loadPackage = Dub.loadPackage; - /** * Returns a fully typed `TestPackageManager` * @@ -302,6 +319,20 @@ { return cast(inout(TestPackageManager)) this.m_packageManager; } + + /** + * Returns a fully-typed `MockPackageSupplier` + * + * This exposes the first (and usually sole) `PackageSupplier` if typed + * as `MockPackageSupplier` so that client can call convenience functions + * on it directly. + */ + public @property inout(MockPackageSupplier) getRegistry() inout + { + // This will not work with `SkipPackageSupplier`. + assert(this.registry !is null, "The registry hasn't been instantiated?"); + return this.registry; + } } /** @@ -335,55 +366,15 @@ /// List of all SCM packages that can be fetched by this instance protected string[GitReference] scm; - /// The virtual filesystem that this PackageManager acts on - protected FSEntry fs; - this(FSEntry filesystem) + this(Filesystem filesystem) { - NativePath local = TestDub.ProjectPath; - NativePath user = TestDub.Paths.userSettings; - NativePath system = TestDub.Paths.systemSettings; - this.fs = filesystem; - super(local, user, system, false); + NativePath local = TestDub.ProjectPath ~ ".dub/packages/"; + NativePath user = TestDub.Paths.userSettings ~ "packages/"; + NativePath system = TestDub.Paths.systemSettings ~ "packages/"; + super(filesystem, local, user, system); } - /// Port of `Project.loadSelections` - SelectedVersions loadSelections(in Package pack) - { - import dub.version_; - import dub.internal.configy.Read; - import dub.internal.dyaml.stdsumtype; - - auto selverfile = (pack.path ~ SelectedVersions.defaultFile); - // No file exists - if (!this.fs.existsFile(selverfile)) - return new SelectedVersions(); - - SelectionsFile selected; - try - { - const content = this.fs.readText(selverfile); - selected = parseConfigString!SelectionsFile( - content, selverfile.toNativeString()); - } - catch (Exception exc) { - logError("Error loading %s: %s", selverfile, exc.message()); - return new SelectedVersions(); - } - - return selected.content.match!( - (Selections!0 s) { - logWarnTag("Unsupported version", - "File %s has fileVersion %s, which is not yet supported by DUB %s.", - selverfile, s.fileVersion, dubVersion); - logWarn("Ignoring selections file. Use a newer DUB version " ~ - "and set the appropriate toolchainRequirements in your recipe file"); - return new SelectedVersions(); - }, - (Selections!1 s) => new SelectedVersions(s), - ); - } - /** * Re-Implementation of `gitClone`. * @@ -406,87 +397,20 @@ this.scm[GitReference(repo)] = dub_json; } - /// - protected override bool existsDirectory(NativePath path) - { - return this.fs.existsDirectory(path); - } + /// Overriden because we currently don't have a way to do dependency + /// injection on `dub.internal.utils : lockFile`. + public override Package store(ubyte[] data, PlacementLocation dest, + in PackageName name, in Version vers) + { + // Most of the code is copied from the base method + assert(!name.sub.length, "Cannot store a subpackage, use main package instead"); + NativePath dstpath = this.getPackagePath(dest, name, vers.toString()); + this.fs.mkdir(dstpath.parentPath()); - /// - protected override void ensureDirectory(NativePath path) - { - this.fs.mkdir(path); - } - - /// - protected override bool existsFile(NativePath path) - { - return this.fs.existsFile(path); - } - - /// - protected override void writeFile(NativePath path, const(ubyte)[] data) - { - return this.fs.writeFile(path, data); - } - - /// - protected override void writeFile(NativePath path, const(char)[] data) - { - return this.fs.writeFile(path, data); - } - - /// - protected override string readText(NativePath path) - { - return this.fs.readText(path); - } - - /// - protected override void removeFile(NativePath path) - { - return this.fs.removeFile(path); - } - - /// - protected override IterateDirDg iterateDirectory(NativePath path) - { - enforce(this.fs.existsDirectory(path), - path.toNativeString() ~ " does not exists or is not a directory"); - auto dir = this.fs.lookup(path); - int iterator(scope int delegate(ref FileInfo) del) { - foreach (c; dir.children) { - FileInfo fi; - fi.name = c.name; - fi.timeModified = c.attributes.modification; - final switch (c.attributes.type) { - case FSEntry.Type.File: - fi.size = c.content.length; - break; - case FSEntry.Type.Directory: - fi.isDirectory = true; - break; - } - if (auto res = del(fi)) - return res; - } - return 0; - } - return &iterator; - } - - /// Ditto - protected override void setTimes(in NativePath path, in SysTime accessTime, - in SysTime modificationTime) - { - this.fs.setTimes(path, accessTime, modificationTime); - } - - /// Ditto - protected override void setAttributes(in NativePath path, uint attributes) - { - this.fs.setAttributes(path, attributes); - } + if (this.fs.existsFile(dstpath)) + return this.getPackage(name, vers, dest); + return this.store_(data, dstpath, name, vers); + } } /** @@ -497,8 +421,16 @@ */ public class MockPackageSupplier : PackageSupplier { - /// Mapping of package name to packages, ordered by `Version` - protected Package[Version][PackageName] pkgs; + /// Internal duplication to avoid having to deserialize the zip content + private struct PkgData { + /// + PackageRecipe recipe; + /// + ubyte[] data; + } + + /// Mapping of package name to package zip data, ordered by `Version` + protected PkgData[Version][PackageName] pkgs; /// URL this was instantiated with protected string url; @@ -509,6 +441,50 @@ this.url = url; } + /** + * Adds a package to this `PackageSupplier` + * + * The registry API bakes in Zip files / binary data. + * When adding a package here, just provide an `Filesystem` + * representing the package directory, which will be converted + * to ZipFile / `ubyte[]` and returned by `fetchPackage`. + * + * This use a delegate approach similar to `TestDub` constructor: + * a delegate must be provided to initialize the package content. + * The delegate will be called once and is expected to contain, + * at its root, the package. + * + * The name of the package will be defined from the recipe file. + * It's version, however, must be provided as parameter. + * + * Params: + * vers = The `Version` of this package. + * dg = A delegate that will populate its parameter with the + * content of the package. + */ + public void add (in Version vers, scope void delegate(scope Filesystem root) dg) + { + scope pkgRoot = new MockFS(); + dg(pkgRoot); + + string recipe = pkgRoot.existsFile(NativePath("dub.json")) ? "dub.json" : null; + if (recipe is null) + recipe = pkgRoot.existsFile(NativePath("dub.sdl")) ? "dub.sdl" : null; + if (recipe is null) + recipe = pkgRoot.existsFile(NativePath("package.json")) ? "package.json" : null; + // Note: If you want to provide an invalid package, override + // [Mock]PackageSupplier. Most tests will expect a well-behaving + // registry so this assert is here to help with writing tests. + assert(recipe !is null, + "No package recipe found: Expected dub.json or dub.sdl"); + auto pkgRecipe = parsePackageRecipe( + pkgRoot.readText(NativePath(recipe)), recipe); + pkgRecipe.version_ = vers.toString(); + const name = PackageName(pkgRecipe.name); + this.pkgs[name][vers] = PkgData( + pkgRecipe, pkgRoot.serializeToZip("%s-%s/".format(name, vers))); + } + /// public override @property string description() { @@ -527,8 +503,7 @@ public override ubyte[] fetchPackage(in PackageName name, in VersionRange dep, bool pre_release) { - assert(0, "%s - fetchPackage not implemented for: %s" - .format(this.url, name.main)); + return this.getBestMatch(name, dep, pre_release).data; } /// @@ -537,14 +512,30 @@ { import dub.recipe.json; - Package match; - if (auto ppkgs = name.main in this.pkgs) - foreach (vers, pkg; *ppkgs) - if ((!vers.isPreRelease || pre_release) && - dep.matches(vers) && - (match is null || match.version_ < vers)) - match = pkg; - return match is null ? Json.init : toJson(match.recipe); + auto match = this.getBestMatch(name, dep, pre_release); + if (!match.data.length) + return Json.init; + auto res = toJson(match.recipe); + return res; + } + + /// + protected PkgData getBestMatch ( + in PackageName name, in VersionRange dep, bool pre_release) + { + auto ppkgs = name.main in this.pkgs; + if (ppkgs is null) + return typeof(return).init; + + PkgData match; + foreach (vers, pr; *ppkgs) + if ((!vers.isPreRelease || pre_release) && + dep.matches(vers) && + (!match.data.length || Version(match.recipe.version_) < vers)) { + match.recipe = pr.recipe; + match.data = pr.data; + } + return match; } /// @@ -554,356 +545,6 @@ } } -/// An abstract filesystem representation -public class FSEntry -{ - /// Type of file system entry - public enum Type : ubyte { - Directory, - File, - } - - /// List FSEntry attributes - protected struct Attributes { - /// The type of FSEntry, see `FSEntry.Type` - public Type type; - /// System-specific attributes for this `FSEntry` - public uint attrs; - /// Last access time - public SysTime access; - /// Last modification time - public SysTime modification; - } - /// Ditto - protected Attributes attributes; - - /// The name of this node - protected string name; - /// The parent of this entry (can be null for the root) - protected FSEntry parent; - union { - /// Children for this FSEntry (with type == Directory) - protected FSEntry[] children; - /// Content for this FDEntry (with type == File) - protected ubyte[] content; - } - - /// Creates a new FSEntry - private this (FSEntry p, Type t, string n) - { - this.attributes.type = t; - this.parent = p; - this.name = n; - } - - /// Create the root of the filesystem, only usable from this module - private this () - { - this.attributes.type = Type.Directory; - } - - /// Get a direct children node, returns `null` if it can't be found - protected inout(FSEntry) lookup(string name) inout return scope - { - assert(!name.canFind('/')); - foreach (c; this.children) - if (c.name == name) - return c; - return null; - } - - /// Get an arbitrarily nested children node - protected inout(FSEntry) lookup(NativePath path) inout return scope - { - auto relp = this.relativePath(path); - if (relp.empty) - return this; - auto segments = relp.bySegment; - if (auto c = this.lookup(segments.front.name)) { - segments.popFront(); - return !segments.empty ? c.lookup(NativePath(segments)) : c; - } - return null; - } - - /** Get the parent `FSEntry` of a `NativePath` - * - * If the parent doesn't exist, an `Exception` will be thrown - * unless `silent` is provided. If the parent path is a file, - * an `Exception` will be thrown regardless of `silent`. - * - * Params: - * path = The path to look up the parent for - * silent = Whether to error on non-existing parent, - * default to `false`. - */ - protected inout(FSEntry) getParent(NativePath path, bool silent = false) - inout return scope - { - // Relative path in the current directory - if (!path.hasParentPath()) - return this; - - // If we're not in the right `FSEntry`, recurse - const parentPath = path.parentPath(); - auto p = this.lookup(parentPath); - enforce(silent || p !is null, - "No such directory: " ~ parentPath.toNativeString()); - enforce(p is null || p.attributes.type == Type.Directory, - "Parent path is not a directory: " ~ parentPath.toNativeString()); - return p; - } - - /// Returns: A path relative to `this.path` - protected NativePath relativePath(NativePath path) const scope - { - assert(!path.absolute() || path.startsWith(this.path), - "Calling relativePath with a differently rooted path"); - return path.absolute() ? path.relativeTo(this.path) : path; - } - - /*+************************************************************************* - - Utility function - - Below this banners are functions that are provided for the convenience - of writing tests for `Dub`. - - ***************************************************************************/ - - /// Prints a visual representation of the filesystem to stdout for debugging - public void print(bool content = false) const scope - { - import std.range : repeat; - static import std.stdio; - - size_t indent; - for (auto p = &this.parent; (*p) !is null; p = &p.parent) - indent++; - // Don't print anything (even a newline) for root - if (this.parent is null) - std.stdio.write('/'); - else - std.stdio.write('|', '-'.repeat(indent), ' ', this.name, ' '); - - final switch (this.attributes.type) { - case Type.Directory: - std.stdio.writeln('(', this.children.length, " entries):"); - foreach (c; this.children) - c.print(content); - break; - case Type.File: - if (!content) - std.stdio.writeln('(', this.content.length, " bytes)"); - else if (this.name.endsWith(".json") || this.name.endsWith(".sdl")) - std.stdio.writeln('(', this.content.length, " bytes): ", - cast(string) this.content); - else - std.stdio.writeln('(', this.content.length, " bytes): ", - this.content); - break; - } - } - - /// Returns: The final destination a specific package needs to be stored in - public static NativePath getPackagePath(in string name_, string vers, - PlacementLocation location = PlacementLocation.user) - { - PackageName name = PackageName(name_); - // Keep in sync with `dub.packagemanager: PackageManager.getPackagePath` - // and `Location.getPackagePath` - NativePath result (in NativePath base) - { - NativePath res = base ~ name.main.toString() ~ vers ~ - name.main.toString(); - res.endsWithSlash = true; - return res; - } - - final switch (location) { - case PlacementLocation.user: - return result(TestDub.Paths.userSettings ~ "packages/"); - case PlacementLocation.system: - return result(TestDub.Paths.systemSettings ~ "packages/"); - case PlacementLocation.local: - return result(TestDub.ProjectPath ~ "/.dub/packages/"); - } - } - - /*+************************************************************************* - - Public filesystem functions - - Below this banners are functions which mimic the behavior of a file - system. - - ***************************************************************************/ - - /// Returns: The `path` of this FSEntry - public NativePath path() const scope - { - if (this.parent is null) - return NativePath("/"); - auto thisPath = this.parent.path ~ this.name; - thisPath.endsWithSlash = (this.attributes.type == Type.Directory); - return thisPath; - } - - /// Implements `mkdir -p`, returns the created directory - public FSEntry mkdir (NativePath path) scope - { - auto relp = this.relativePath(path); - // Check if the child already exists - auto segments = relp.bySegment; - auto child = this.lookup(segments.front.name); - if (child is null) { - child = new FSEntry(this, Type.Directory, segments.front.name); - this.children ~= child; - } - // Recurse if needed - segments.popFront(); - return !segments.empty ? child.mkdir(NativePath(segments)) : child; - } - - /// Checks the existence of a file - public bool existsFile (NativePath path) const scope - { - auto entry = this.lookup(path); - return entry !is null && entry.attributes.type == Type.File; - } - - /// Checks the existence of a directory - public bool existsDirectory (NativePath path) const scope - { - auto entry = this.lookup(path); - return entry !is null && entry.attributes.type == Type.Directory; - } - - /// Reads a file, returns the content as `ubyte[]` - public ubyte[] readFile (NativePath path) const scope - { - auto entry = this.lookup(path); - enforce(entry.attributes.type == Type.File, "Trying to read a directory"); - return entry.content.dup; - } - - /// Reads a file, returns the content as text - public string readText (NativePath path) const scope - { - import std.utf : validate; - - auto entry = this.lookup(path); - enforce(entry.attributes.type == Type.File, "Trying to read a directory"); - // Ignore BOM: If it's needed for a test, add support for it. - validate(cast(const(char[])) entry.content); - return cast(string) entry.content.idup(); - } - - /// Write to this file - public void writeFile (NativePath path, const(char)[] data) scope - { - this.writeFile(path, data.representation); - } - - /// Ditto - public void writeFile (NativePath path, const(ubyte)[] data) scope - { - enforce(!path.endsWithSlash(), - "Cannot write to directory: " ~ path.toNativeString()); - if (auto file = this.lookup(path)) { - // If the file already exists, override it - enforce(file.attributes.type == Type.File, - "Trying to write to directory: " ~ path.toNativeString()); - file.content = data.dup; - } else { - auto p = this.getParent(path); - auto file = new FSEntry(p, Type.File, path.head.name()); - file.content = data.dup; - p.children ~= file; - } - } - - /** Remove a file - * - * Always error if the target is a directory. - * Does not error if the target does not exists - * and `force` is set to `true`. - * - * Params: - * path = Path to the file to remove - * force = Whether to ignore non-existing file, - * default to `false`. - */ - public void removeFile (NativePath path, bool force = false) - { - import std.algorithm.searching : countUntil; - - assert(!path.empty, "Empty path provided to `removeFile`"); - enforce(!path.endsWithSlash(), - "Cannot remove file with directory path: " ~ path.toNativeString()); - auto p = this.getParent(path, force); - const idx = p.children.countUntil!(e => e.name == path.head.name()); - if (idx < 0) { - enforce(force, - "removeFile: No such file: " ~ path.toNativeString()); - } else { - enforce(p.children[idx].attributes.type == Type.File, - "removeFile called on a directory: " ~ path.toNativeString()); - p.children = p.children[0 .. idx] ~ p.children[idx + 1 .. $]; - } - } - - /** Remove a directory - * - * Remove an existing empty directory. - * If `force` is set to `true`, no error will be thrown - * if the directory is empty or non-existing. - * - * Params: - * path = Path to the directory to remove - * force = Whether to ignore non-existing / non-empty directories, - * default to `false`. - */ - public void removeDir (NativePath path, bool force = false) - { - import std.algorithm.searching : countUntil; - - assert(!path.empty, "Empty path provided to `removeFile`"); - auto p = this.getParent(path, force); - const idx = p.children.countUntil!(e => e.name == path.head.name()); - if (idx < 0) { - enforce(force, - "removeDir: No such directory: " ~ path.toNativeString()); - } else { - enforce(p.children[idx].attributes.type == Type.Directory, - "removeDir called on a file: " ~ path.toNativeString()); - enforce(force || p.children[idx].children.length == 0, - "removeDir called on non-empty directory: " ~ path.toNativeString()); - p.children = p.children[0 .. idx] ~ p.children[idx + 1 .. $]; - } - } - - /// Implement `std.file.setTimes` - public void setTimes(in NativePath path, in SysTime accessTime, - in SysTime modificationTime) - { - auto e = this.lookup(path); - enforce(e !is null, - "setTimes: No such file or directory: " ~ path.toNativeString()); - e.attributes.access = accessTime; - e.attributes.modification = modificationTime; - } - - /// Implement `std.file.setAttributes` - public void setAttributes(in NativePath path, uint attributes) - { - auto e = this.lookup(path); - enforce(e !is null, - "setTimes: No such file or directory: " ~ path.toNativeString()); - e.attributes.attrs = attributes; - } -} - /** * Convenience function to write a package file * @@ -911,19 +552,45 @@ * package name and version. * * Params: - * root = The root FSEntry + * root = The root Filesystem * name = The package name (typed as string for convenience) * vers = The package version * recipe = The text of the package recipe * fmt = The format used for `recipe` (default to JSON) * location = Where to place the package (default to user location) */ -public void writePackageFile (FSEntry root, in string name, in string vers, +public void writePackageFile (Filesystem root, in string name, in string vers, in string recipe, in PackageFormat fmt = PackageFormat.json, in PlacementLocation location = PlacementLocation.user) { - const path = FSEntry.getPackagePath(name, vers, location); - root.mkdir(path).writeFile( - NativePath(fmt == PackageFormat.json ? "dub.json" : "dub.sdl"), + const path = getPackagePath(name, vers, location); + root.mkdir(path); + root.writeFile( + path ~ (fmt == PackageFormat.json ? "dub.json" : "dub.sdl"), recipe); } + +/// Returns: The final destination a specific package needs to be stored in +public static NativePath getPackagePath(in string name_, string vers, + PlacementLocation location = PlacementLocation.user) +{ + PackageName name = PackageName(name_); + // Keep in sync with `dub.packagemanager: PackageManager.getPackagePath` + // and `Location.getPackagePath` + NativePath result (in NativePath base) + { + NativePath res = base ~ name.main.toString() ~ vers ~ + name.main.toString(); + res.endsWithSlash = true; + return res; + } + + final switch (location) { + case PlacementLocation.user: + return result(TestDub.Paths.userSettings ~ "packages/"); + case PlacementLocation.system: + return result(TestDub.Paths.systemSettings ~ "packages/"); + case PlacementLocation.local: + return result(TestDub.ProjectPath ~ "/.dub/packages/"); + } +} diff --git a/source/dub/test/dependencies.d b/source/dub/test/dependencies.d index 180f31d..4b31f7d 100644 --- a/source/dub/test/dependencies.d +++ b/source/dub/test/dependencies.d @@ -31,7 +31,7 @@ // Ensure that simple dependencies get resolved correctly unittest { - scope dub = new TestDub((scope FSEntry root) { + scope dub = new TestDub((scope Filesystem root) { root.writeFile(TestDub.ProjectPath ~ "dub.sdl", `name "a" version "1.0.0" dependency "b" version="*" @@ -55,7 +55,7 @@ // Test that indirect dependencies get resolved correctly unittest { - scope dub = new TestDub((scope FSEntry root) { + scope dub = new TestDub((scope Filesystem root) { root.writeFile(TestDub.ProjectPath ~ "dub.sdl", `name "a" dependency "b" version="*"`); root.writePackageFile("b", "1.0.0", `name "b" @@ -77,7 +77,7 @@ // Simple diamond dependency unittest { - scope dub = new TestDub((scope FSEntry root) { + scope dub = new TestDub((scope Filesystem root) { root.writeFile(TestDub.ProjectPath ~ "dub.sdl", `name "a" dependency "b" version="*" dependency "c" version="*"`); @@ -105,7 +105,7 @@ // Missing dependencies trigger an error unittest { - scope dub = new TestDub((scope FSEntry root) { + scope dub = new TestDub((scope Filesystem root) { root.writeFile(TestDub.ProjectPath ~ "dub.sdl", `name "a" dependency "b" version="*"`); }); diff --git a/source/dub/test/other.d b/source/dub/test/other.d index 241fb0f..be406a2 100644 --- a/source/dub/test/other.d +++ b/source/dub/test/other.d @@ -21,7 +21,7 @@ const Template = `{"name": "%s", "version": "1.0.0", "dependencies": { "dep1": { "repository": "%s", "version": "%s" }}}`; - scope dub = new TestDub((scope FSEntry fs) { + scope dub = new TestDub((scope Filesystem fs) { // Invalid URL, valid hash fs.writePackageFile("a", "1.0.0", Template.format("a", "git+https://nope.nope", ValidHash)); // Valid URL, invalid hash @@ -53,7 +53,7 @@ { const AddPathDir = TestDub.Paths.temp ~ "addpath/"; const BDir = AddPathDir ~ "b/"; - scope dub = new TestDub((scope FSEntry root) { + scope dub = new TestDub((scope Filesystem root) { root.writeFile(TestDub.ProjectPath ~ "dub.json", `{ "name": "a", "dependencies": { "b": "~>1.0" } }`); @@ -79,3 +79,44 @@ const actualDir = newDub.project.getDependency("b", true).path(); assert(actualDir == BDir, actualDir.toNativeString()); } + +// Check that SCM-only dependencies don't lead to a scan of the FS +unittest +{ + const ValidURL = `git+https://example.com/dlang/dub`; + // Taken from a commit in the dub repository + const ValidHash = "54339dff7ce9ec24eda550f8055354f712f15800"; + const Template = `{"name": "%s", "version": "1.0.0", "dependencies": { +"dep1": { "repository": "%s", "version": "%s" }}}`; + + scope dub = new TestDub((scope Filesystem fs) { + // This should never be read + fs.writePackageFile("poison", "1.0.0", `poison`); + fs.writeFile(TestDub.ProjectPath ~ "dub.json", + `{ "name": "a", "dependencies": {"b": { "repository": "` ~ + ValidURL ~ `", "version": "` ~ ValidHash ~ `" }} }`); + }); + dub.packageManager.addTestSCMPackage( + Repository(ValidURL, ValidHash), `{"name":"b"}`); + + dub.loadPackage(); + assert(dub.project.hasAllDependencies()); +} + +// Check that a simple build does not lead to the cache being scanned +unittest +{ + scope dub = new TestDub((scope Filesystem fs) { + // This should never be read + fs.writePackageFile("b", "1.0.0", `poison`); + fs.writePackageFile("b", "1.1.0", `poison`); + // Dependency resolution may trigger scan, so we need a selections file + fs.writeFile(TestDub.ProjectPath ~ "dub.json", + `{ "name": "a", "dependencies": {"b":"~>1.0"}}`); + fs.writeFile(TestDub.ProjectPath ~ "dub.selections.json", + `{"fileVersion":1,"versions":{"b":"1.0.4"}}`); + fs.writePackageFile("b", "1.0.4", `{"name":"b","version":"1.0.4"}`); + }); + dub.loadPackage(); + assert(dub.project.hasAllDependencies()); +} diff --git a/source/dub/test/selections_from_parent_dir.d b/source/dub/test/selections_from_parent_dir.d new file mode 100644 index 0000000..1a7c418 --- /dev/null +++ b/source/dub/test/selections_from_parent_dir.d @@ -0,0 +1,152 @@ +/******************************************************************************* + + Test inheritable flag of selections file + + Selections files can have an `inheritable` flag that is used to have + a central selections file, e.g. in the case of a monorepo. + +*******************************************************************************/ + +module dub.test.selections_from_parent_dir; + +version (unittest): + +import dub.test.base; +import std.string : replace; + +// dub.selections.json can be inherited from parent directories +unittest +{ + const pkg1Dir = TestDub.ProjectPath ~ "pkg1"; + const pkg2Dir = TestDub.ProjectPath ~ "pkg2"; + const path = TestDub.ProjectPath ~ "dub.selections.json"; + const dubSelectionsJsonContent = `{ + "fileVersion": 1, + "inheritable": true, + "versions": { + "pkg1": {"path":"pkg1"} + } +} +`; + + scope dub = new TestDub((scope Filesystem fs) { + fs.mkdir(pkg1Dir); + fs.writeFile(pkg1Dir ~ "dub.sdl", `name "pkg1" +targetType "none"`); + fs.mkdir(pkg2Dir); + fs.writeFile(pkg2Dir ~ "dub.sdl", `name "pkg2" +targetType "library" + +# don't specify a path, require inherited dub.selections.json to make it path-based (../pkg1) +dependency "pkg1" version="*"`); + + // important: dub.selections.json in *parent* directory + fs.writeFile(path, dubSelectionsJsonContent); + }, pkg2Dir.toNativeString()); // pkg2 is our root package + + dub.loadPackage(); + assert(dub.project.hasAllDependencies()); + // the relative path should have been adjusted (`pkg1` => `../pkg1`) + assert(dub.project.selections.getSelectedVersion(PackageName("pkg1")).path == NativePath("../pkg1")); + + // invoking `dub upgrade` for the pkg2 root package should generate a local dub.selections.json, + // leaving the inherited one untouched + dub.upgrade(UpgradeOptions.select); + const nestedPath = pkg2Dir ~ "dub.selections.json"; + assert(dub.fs.existsFile(nestedPath)); + assert(dub.fs.readFile(path) == dubSelectionsJsonContent, + "Inherited dub.selections.json modified after dub upgrade!"); + const nestedContent = dub.fs.readText(nestedPath); + assert(nestedContent == dubSelectionsJsonContent.replace(`{"path":"pkg1"}`, `{"path":"../pkg1"}`), + "Unexpected nestedContent:\n" ~ nestedContent); +} + +// a non-inheritable dub.selections.json breaks the inheritance chain +unittest +{ + const root = TestDub.ProjectPath ~ "root"; + const root_a = root ~ "a"; + const root_a_b = root_a ~ "b"; + + scope dub_ = new TestDub((scope Filesystem fs) { + // inheritable root/dub.selections.json + fs.mkdir(root); + fs.writeFile(root ~ "dub.selections.json", `{ + "fileVersion": 1, + "inheritable": true, + "versions": { + "dub": "1.38.0" + } +} +`); + // non-inheritable root/a/dub.selections.json + fs.mkdir(root_a); + fs.writeFile(root_a ~ "dub.selections.json", `{ + "fileVersion": 1, + "versions": { + "dub": "1.37.0" + } +} +`); + // We need packages for `loadPackage` + fs.mkdir(root_a_b); + fs.writeFile(root_a_b ~ `dub.json`, + `{"name":"ab","dependencies":{"dub":"~>1.0"}}`); + fs.writeFile(root_a ~ `dub.json`, + `{"name":"a","dependencies":{"dub":"~>1.0"}}`); + fs.writeFile(root ~ `dub.json`, + `{"name":"r","dependencies":{"dub":"~>1.0"}}`); + fs.writePackageFile(`dub`, `1.37.0`, `{"name":"dub","version":"1.37.0"}`); + fs.writePackageFile(`dub`, `1.38.0`, `{"name":"dub","version":"1.38.0"}`); + }); + + // no selections for root/a/b/ + { + auto dub = dub_.newTest(); + const result = dub.packageManager.readSelections(root_a_b); + assert(result.isNull()); + dub.loadPackage(root_a_b); + assert(!dub.project.hasAllDependencies()); + } + + // local selections for root/a/ + { + auto dub = dub_.newTest(); + const result = dub.packageManager.readSelections(root_a); + assert(!result.isNull()); + assert(result.get().absolutePath == root_a ~ "dub.selections.json"); + assert(!result.get().selectionsFile.inheritable); + dub.loadPackage(root_a); + assert(dub.project.hasAllDependencies()); + assert(dub.project.dependencies()[0].name == "dub"); + assert(dub.project.dependencies()[0].version_ == Version("1.37.0")); + } + + // local selections for root/ + { + auto dub = dub_.newTest(); + const result = dub.packageManager.readSelections(root); + assert(!result.isNull()); + assert(result.get().absolutePath == root ~ "dub.selections.json"); + assert(result.get().selectionsFile.inheritable); + dub.loadPackage(root); + assert(dub.project.hasAllDependencies()); + assert(dub.project.dependencies()[0].name == "dub"); + assert(dub.project.dependencies()[0].version_ == Version("1.38.0")); + } + + // after removing non-inheritable root/a/dub.selections.json: inherited root selections for root/a/b/ + { + auto dub = dub_.newTest((scope Filesystem fs) { + fs.removeFile(root_a ~ "dub.selections.json"); + }); + const result = dub.packageManager.readSelections(root_a_b); + assert(!result.isNull()); + assert(result.get().absolutePath == root ~ "dub.selections.json"); + assert(result.get().selectionsFile.inheritable); + dub.loadPackage(root_a_b); + assert(dub.project.hasAllDependencies()); + assert(dub.project.dependencies()[0].name == "dub"); + assert(dub.project.dependencies()[0].version_ == Version("1.38.0")); + } +} diff --git a/source/dub/test/subpackages.d b/source/dub/test/subpackages.d index a84ae83..a56ac0c 100644 --- a/source/dub/test/subpackages.d +++ b/source/dub/test/subpackages.d @@ -17,7 +17,7 @@ /// Test of the PackageManager APIs unittest { - scope dub = new TestDub((scope FSEntry root) { + scope dub = new TestDub((scope Filesystem root) { root.writeFile(TestDub.ProjectPath ~ "dub.json", `{ "name": "a", "dependencies": { "b:a": "~>1.0", "b:b": "~>1.0" } }`); root.writePackageFile("b", "1.0.0", diff --git a/test/5-convert/dub.sdl b/test/5-convert/dub.sdl index cd3a039..a8753ce 100644 --- a/test/5-convert/dub.sdl +++ b/test/5-convert/dub.sdl @@ -7,7 +7,7 @@ license "BSD 2-clause" x:ddoxFilterArgs "dfa1" "dfa2" x:ddoxTool "ddoxtool" -dependency "describe-dependency-1:sub1" version=">=0.0.0" +dependency "describe-dependency-1:sub1" version="*" targetType "sourceLibrary" subConfiguration "describe-dependency-1:sub1" "library" dflags "--another-dflag" diff --git a/test/prefer-add-path-packages.sh b/test/prefer-add-path-packages.sh deleted file mode 100755 index af8bc05..0000000 --- a/test/prefer-add-path-packages.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname ${BASH_SOURCE[0]})/common.sh - -TMPDIR=$CURR_DIR/tmp-add-path - -PACK_PATH="$CURR_DIR"/issue2262-exact-cached-version-match - -# make sure that there are no left-over selections files or temp directory -rm -f $PACK_PATH/dub.selections.json -rm -rf $TMPDIR - -# make sure that there are no cached versions of the dependency -$DUB remove gitcompatibledubpackage@* -n || true - -# build normally, should select 1.0.4 -if ! ${DUB} build --root $PACK_PATH | grep "gitcompatibledubpackage 1\.0\.4:"; then - die $LINENO 'The initial build failed.' -fi - -# clone gitcompatibledubpackage and check out 1.0.4+commit.2.ccb31bf -mkdir $TMPDIR -$DUB add-path $TMPDIR -cd $TMPDIR -git clone https://github.com/dlang-community/gitcompatibledubpackage.git -cd gitcompatibledubpackage -git checkout -q ccb31bf6a655437176ec02e04c2305a8c7c90d67 -cd ../.. - -if ! $DUB list | grep "gitcompatibledubpackage 1\.0\.4+commit.2.gccb31bf"; then - $DUB remove-path $TMPDIR - die $LINENO 'Cloned package was not found in search path' -fi - -# should pick up the cloned package instead of the cached one now -if ! ${DUB} build --root $PACK_PATH | grep "gitcompatibledubpackage 1\.0\.4+commit.2.gccb31bf:"; then - $DUB remove-path $TMPDIR - die $LINENO 'Did not pick up the add-path package.' -fi - -# clean up -$DUB remove-path $TMPDIR -rm -f $PACK_PATH/dub.selections.json -rm -rf $TMPDIR diff --git a/test/removed-dub-obj.sh b/test/removed-dub-obj.sh index 1334f8b..d767b43 100755 --- a/test/removed-dub-obj.sh +++ b/test/removed-dub-obj.sh @@ -3,19 +3,14 @@ . $(dirname "${BASH_SOURCE[0]}")/common.sh cd ${CURR_DIR}/removed-dub-obj -DUB_CACHE_PATH="$HOME/.dub/cache/removed-dub-obj/" +DUB_CACHE_PATH="$HOME/.dub/cache/removed-dub-obj" -rm -rf $DUB_CACHE_PATH +rm -rf "$DUB_CACHE_PATH" ${DUB} build --compiler=${DC} -[ -d "$DUB_CACHE_PATH/obj" ] && die $LINENO "$DUB_CACHE_PATH/obj was found" +[ -d "$DUB_CACHE_PATH" ] || die $LINENO "$DUB_CACHE_PATH not found" -if [[ ${DC} == *"ldc"* ]]; then - if [ ! -f $DUB_CACHE_PATH/~master/build/library-*/obj/test.o* ]; then - ls -lR $DUB_CACHE_PATH - die $LINENO '$DUB_CACHE_PATH/~master/build/library-*/obj/test.o* was not found' - fi -fi - -exit 0 +numObjectFiles=$(find "$DUB_CACHE_PATH" -type f -iname '*.o*' | wc -l) +# note: fails with LDC < v1.1 +[ "$numObjectFiles" -eq 0 ] || die $LINENO "Found left-over object files in $DUB_CACHE_PATH" diff --git a/test/run-unittest.d b/test/run-unittest.d index 71ee3f4..89c636e 100644 --- a/test/run-unittest.d +++ b/test/run-unittest.d @@ -10,12 +10,8 @@ int main(string[] args) { - import std.algorithm : among, endsWith; - import std.file : dirEntries, DirEntry, exists, getcwd, readText, SpanMode; - import std.format : format; - import std.stdio : File, writeln; - import std.path : absolutePath, buildNormalizedPath, baseName, dirName; - import std.process : environment, spawnProcess, wait; + import std.algorithm, std.file, std.format, std.stdio, std.path, std.process, std.string; + alias ProcessConfig = std.process.Config; //** if [ -z ${DUB:-} ]; then //** die $LINENO 'Variable $DUB must be defined to run the tests.' @@ -46,12 +42,15 @@ //** DC_BIN=$(basename "$DC") //** CURR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) //** FRONTEND="${FRONTEND:-}" - const dc_bin = baseName(dc); + const dc_bin = baseName(dc).stripExtension; const curr_dir = __FILE_FULL_PATH__.dirName(); - const frontend = environment.get("FRONTEND", ""); + const frontend = environment.get("FRONTEND", __VERSION__.format!"%04d"); //** if [ "$#" -gt 0 ]; then FILTER=$1; else FILTER=".*"; fi auto filter = (args.length > 1) ? args[1] : "*"; + version (linux) auto os = "linux"; + version (Windows) auto os = "windows"; + version (OSX) auto os = "osx"; version (Posix) { @@ -62,25 +61,29 @@ //** log "Running $script..." //** DUB=$DUB DC=$DC CURR_DIR="$CURR_DIR" $script || logError "Script failure." //** done - foreach(DirEntry script; dirEntries(curr_dir, (args.length > 1) ? args[1] : "*.sh", SpanMode.shallow)) + foreach(DirEntry script; dirEntries(curr_dir, "*.sh", SpanMode.shallow)) { + if (!script.name.baseName.globMatch(filter)) continue; if (!script.name.endsWith(".sh")) continue; if (baseName(script.name).among("run-unittest.sh", "common.sh")) continue; const min_frontend = script.name ~ ".min_frontend"; - if (exists(min_frontend) && frontend.length && frontend < min_frontend.readText) continue; + if (exists(min_frontend) && frontend.length && cmp(frontend, min_frontend.readText) < 0) continue; log("Running " ~ script ~ "..."); - if (spawnProcess(script.name, ["DUB":dub, "DC":dc, "CURR_DIR":curr_dir]).wait) + if (spawnShell(script.name, ["DUB":dub, "DC":dc, "CURR_DIR":curr_dir]).wait) logError("Script failure."); + else + log(script.name.baseName, " status: Ok"); } } - foreach (DirEntry script; dirEntries(curr_dir, (args.length > 1) ? args[1] : "*.script.d", SpanMode.shallow)) + foreach (DirEntry script; dirEntries(curr_dir, "*.script.d", SpanMode.shallow)) { + if (!script.name.baseName.globMatch(filter)) continue; if (!script.name.endsWith(".d")) continue; const min_frontend = script.name ~ ".min_frontend"; - if (frontend.length && exists(min_frontend) && frontend < min_frontend.readText) continue; + if (frontend.length && exists(min_frontend) && cmp(frontend, min_frontend.readText) < 0) continue; log("Running " ~ script ~ "..."); if (spawnProcess([dub, script.name], ["DUB":dub, "DC":dc, "CURR_DIR":curr_dir]).wait) logError("Script failure."); @@ -88,5 +91,100 @@ log(script.name, " status: Ok"); } + //for pack in $(ls -d $CURR_DIR/*/); do + foreach (DirEntry pack; dirEntries(curr_dir, SpanMode.shallow)) + { + //if [[ ! "$pack" =~ $FILTER ]]; then continue; fi + if (!pack.name.baseName.globMatch(filter)) continue; + if (!pack.isDir || pack.name.baseName.startsWith(".")) continue; + if (!pack.name.buildPath("dub.json").exists && !pack.name.buildPath("dub.sdl").exists && !pack.name.buildPath("package.json").exists) continue; + //if [ -e $pack/.min_frontend ] && [ ! -z "$FRONTEND" -a "$FRONTEND" \< $(cat $pack/.min_frontend) ]; then continue; fi + if (pack.name.buildPath(".min_frontend").exists && cmp(frontend, pack.name.buildPath(".min_frontend").readText) < 0) continue; + + //#First we build the packages + //if [ ! -e $pack/.no_build ] && [ ! -e $pack/.no_build_$DC_BIN ]; then # For sourceLibrary + bool build = (!pack.name.buildPath(".no_build").exists + && !pack.name.buildPath(".no_build_" ~ dc_bin).exists + && !pack.name.buildPath(".no_build_" ~ os).exists); + if (build) + { + //build=1 + //if [ -e $pack/.fail_build ]; then + // log "Building $pack, expected failure..." + // $DUB build --force --root=$pack --compiler=$DC 2>/dev/null && logError "Error: Failure expected, but build passed." + //else + // log "Building $pack..." + // $DUB build --force --root=$pack --compiler=$DC || logError "Build failure." + //fi + //if [ -e $pack/.fail_build ]; then + if (pack.name.buildPath(".fail_build").exists) + { + log("Building " ~ pack.name.baseName ~ ", expected failure..."); + if (spawnProcess([dub, "build", "--force", "--compiler", dc], ["DUB":dub, "DC":dc, "CURR_DIR":curr_dir], ProcessConfig.none, pack.name).wait) + log(pack.name.baseName, " status: Ok"); + else + logError("Failure expected, but build passed."); + } + else + { + log("Building ", pack.name.baseName, "..."); + if (spawnProcess([dub, "build", "--force", "--compiler", dc], ["DUB":dub, "DC":dc, "CURR_DIR":curr_dir], ProcessConfig.none, pack.name).wait) + logError("Script failure."); + else + log(pack.name.baseName, " status: Ok"); + } + } + //else + // build=0 + //fi + + //# We run the ones that are supposed to be run + //if [ $build -eq 1 ] && [ ! -e $pack/.no_run ] && [ ! -e $pack/.no_run_$DC_BIN ]; then + // log "Running $pack..." + // $DUB run --force --root=$pack --compiler=$DC || logError "Run failure." + //fi + if (build + && !pack.name.buildPath(".no_run").exists + && !pack.name.buildPath(".no_run_" ~ dc_bin).exists + && !pack.name.buildPath(".no_run_" ~ os).exists) + { + log("Running ", pack.name.baseName, "..."); + if (spawnProcess([dub, "run", "--force", "--compiler", dc], ["DUB":dub, "DC":dc, "CURR_DIR":curr_dir], ProcessConfig.none, pack.name).wait) + logError("Run failure."); + else + log(pack.name.baseName, " status: Ok"); + } + + //# Finally, the unittest part + //if [ $build -eq 1 ] && [ ! -e $pack/.no_test ] && [ ! -e $pack/.no_test_$DC_BIN ]; then + // log "Testing $pack..." + // $DUB test --force --root=$pack --compiler=$DC || logError "Test failure." + //fi + if (build + && !pack.name.buildPath(".no_test").exists + && !pack.name.buildPath(".no_test_" ~ dc_bin).exists + && !pack.name.buildPath(".no_test_" ~ os).exists) + { + log("Testing ", pack.name.baseName, "..."); + if (spawnProcess([dub, "test", "--force", "--root", pack.name, "--compiler", dc], ["DUB":dub, "DC":dc, "CURR_DIR":curr_dir]).wait) + logError("Test failure."); + else + log(pack.name.baseName, " status: Ok"); + } + //done + } + + //echo + //echo 'Testing summary:' + //cat $(dirname "${BASH_SOURCE[0]}")/test.log + writeln(); + writeln("Testing summary:"); + auto logLines = readText("test.log").splitLines; + foreach (line; logLines) + writeln(line); + auto errCnt = logLines.count!(a => a.startsWith("[ERROR]")); + auto passCnt = logLines.count!(a => a.startsWith("[INFO]") && a.endsWith("status: Ok")); + writeln(passCnt , "/", errCnt + passCnt, " tests were successed."); + return any_errors; }