diff --git a/changelog/git-paths.dd b/changelog/git-paths.dd new file mode 100644 index 0000000..33447b7 --- /dev/null +++ b/changelog/git-paths.dd @@ -0,0 +1,16 @@ +Support dependencies as git url with exact commit + +Git repositories can be directly used by dub as dependencies. + +dub.json: +------- +{ + "name": "git-dependency", + "dependencies": { + "gitcompatibledubpackage": { + "repository": "git+https://github.com/dlang-community/gitcompatibledubpackage.git", + "version": "ccb31bf6a655437176ec02e04c2305a8c7c90d67" + } + } +} +------- diff --git a/source/dub/dependency.d b/source/dub/dependency.d index f16edcc..c9b1515 100644 --- a/source/dub/dependency.d +++ b/source/dub/dependency.d @@ -34,7 +34,6 @@ Dependency spec; } - /** Represents a dependency specification. @@ -56,6 +55,7 @@ NativePath m_path; bool m_optional = false; bool m_default = false; + Repository m_repository; } /// A Dependency, which matches every valid version. @@ -93,11 +93,31 @@ m_path = path; } + /** Constructs a new dependency specification that matches a specific + Git reference. + */ + this(Repository repository, string spec) { + this.versionSpec = spec; + this.repository = repository; + } + /// If set, overrides any version based dependency selection. @property void path(NativePath value) { m_path = value; } /// ditto @property NativePath path() const { return m_path; } + /// If set, overrides any version based dependency selection. + @property void repository(Repository value) + { + m_repository = value; + } + + /// ditto + @property Repository repository() const + { + return m_repository; + } + /// Determines if the dependency is required or optional. @property bool optional() const { return m_optional; } /// ditto @@ -111,6 +131,9 @@ /// Returns true $(I iff) the version range only matches a specific version. @property bool isExactVersion() const { return m_versA == m_versB; } + /// Determines whether it is a Git dependency. + @property bool isSCM() const { return !repository.empty; } + /// Returns the exact version matched by the version range. @property Version version_() const { enforce(m_versA == m_versB, "Dependency "~this.versionSpec~" is no exact version."); @@ -167,7 +190,7 @@ ves = ves[1..$].expandVersion; m_versA = Version(ves); m_versB = Version(bumpIncompatibleVersion(ves) ~ "-0"); - } else if (ves[0] == Version.branchPrefix) { + } else if (ves[0] == Version.branchPrefix || ves.isGitHash) { m_inclusiveA = true; m_inclusiveB = true; m_versA = m_versB = Version(ves); @@ -218,7 +241,6 @@ string r; if (this == invalid) return "invalid"; - if (m_versA == m_versB && m_inclusiveA && m_inclusiveB) { // Special "==" case if (m_versA == Version.masterBranch) return "~master"; @@ -272,7 +294,12 @@ */ string toString()() const { - auto ret = versionSpec; + string ret; + + if (!repository.empty) { + ret ~= repository.toString~"#"; + } + ret ~= versionSpec; if (optional) { if (default_) ret ~= " (optional, default)"; else ret ~= " (optional)"; @@ -296,12 +323,13 @@ Json toJson() const @trusted { // NOTE Path and Json is @system in vibe.d 0.7.x and in the compatibility layer Json json; - if( path.empty && !optional ){ + if( path.empty && repository.empty && !optional ){ json = Json(this.versionSpec); } else { json = Json.emptyObject; json["version"] = this.versionSpec; if (!path.empty) json["path"] = path.toString(); + if (!repository.empty) json["repository"] = repository.toString; if (optional) json["optional"] = true; if (default_) json["default"] = true; } @@ -316,6 +344,15 @@ assert(d.toJson() == Json("1.0.0"), "Failed: " ~ d.toJson().toPrettyString()); } + @trusted unittest { + Dependency dependency = Dependency(Repository("git+http://localhost"), "1.0.0"); + Json expected = Json([ + "repository": Json("git+http://localhost"), + "version": Json("1.0.0") + ]); + assert(dependency.toJson() == expected, "Failed: " ~ dependency.toJson().toPrettyString()); + } + /** Constructs a new `Dependency` from its JSON representation. See `toJson` for a description of the JSON format. @@ -330,6 +367,12 @@ dep = Dependency.any; dep.path = NativePath(verspec["path"].get!string); + } else if (auto repository = "repository" in verspec) { + enforce("version" in verspec, "No version field specified!"); + enforce(repository.length > 0, "No repository field specified!"); + + dep = Dependency(Repository(repository.get!string), + verspec["version"].get!string); } else { enforce("version" in verspec, "No version field specified!"); auto ver = verspec["version"].get!string; @@ -411,6 +454,7 @@ A specification is valid if it can match at least one version. */ bool valid() const { + if (this.isSCM) return true; return m_versA <= m_versB && doCmp(m_inclusiveA && m_inclusiveB, m_versA, m_versB); } @@ -440,6 +484,7 @@ /// ditto bool matches(ref const(Version) v) const { if (this.matchesAny) return true; + if (this.isSCM) return true; //logDebug(" try match: %s with: %s", v, this); // Master only matches master if(m_versA.isBranch) { @@ -463,6 +508,13 @@ */ Dependency merge(ref const(Dependency) o) const { + if (this.isSCM) { + if (!o.isSCM) return this; + if (this.m_versA == o.m_versA) return this; + return invalid; + } + if (o.isSCM) return o; + if (this.matchesAny) return o; if (o.matchesAny) return this; if (m_versA.isBranch != o.m_versA.isBranch) return invalid; @@ -677,6 +729,80 @@ assert(Dependency("^1.2").versionSpec == "~>1.2"); // equivalent; prefer ~> } +/** + Represents an SCM repository. +*/ +struct Repository +{ + private string m_remote; + + private Kind m_kind; + + enum Kind + { + git, + } + + /** + Params: + remote = Repository remote. + */ + this(string remote) + { + if (remote.startsWith("git+")) + { + m_remote = remote["git+".length .. $]; + m_kind = Kind.git; + } + else + { + throw new Exception("Unsupported repository type"); + } + } + + string toString() nothrow pure @safe + { + if (empty) return null; + string kindRepresentation; + + final switch (kind) + { + case Kind.git: + kindRepresentation = "git"; + } + return kindRepresentation~"+"~remote; + } + + /** + Returns: + Repository URL or path. + */ + @property string remote() @nogc nothrow pure @safe + in { assert(m_remote !is null); } + body + { + return m_remote; + } + + /** + Returns: + Repository type. + */ + @property Kind kind() @nogc nothrow pure @safe + { + return m_kind; + } + + /** + Returns: + Whether the repository was initialized with an URL or path. + */ + @property bool empty() const @nogc nothrow pure @safe + { + return m_remote.empty; + } +} + /** Represents a version in semantic version format, or a branch identifier. @@ -705,7 +831,7 @@ this(string vers) { enforce(vers.length > 1, "Version strings must not be empty."); - if (vers[0] != branchPrefix && vers.ptr !is UNKNOWN_VERS.ptr) + if (vers[0] != branchPrefix && !vers.isGitHash && vers.ptr !is UNKNOWN_VERS.ptr) enforce(vers.isValidVersion(), "Invalid SemVer format: " ~ vers); m_version = vers; } @@ -719,6 +845,9 @@ bool opEquals(const Version oth) const { return opCmp(oth) == 0; } + /// Tests if this represents a hash instead of a version. + @property bool isSCM() const { return m_version.isGitHash; } + /// Tests if this represents a branch instead of a version. @property bool isBranch() const { return m_version.length > 0 && m_version[0] == branchPrefix; } @@ -730,7 +859,7 @@ Note that branches are always considered pre-release versions. */ @property bool isPreRelease() const { - if (isBranch) return true; + if (isBranch || isSCM) return true; return isPreReleaseVersion(m_version); } @@ -749,6 +878,13 @@ if (isUnknown || other.isUnknown) { throw new Exception("Can't compare unknown versions! (this: %s, other: %s)".format(this, other)); } + + if (isSCM || other.isSCM) { + if (!isSCM) return -1; + if (!other.isSCM) return 1; + return m_version == other.m_version ? 0 : 1; + } + if (isBranch || other.isBranch) { if(m_version == other.m_version) return 0; if (!isBranch) return 1; @@ -790,6 +926,7 @@ assert(a == b, "a == b with a:'1.0.0', b:'1.0.0' failed"); b = Version("2.0.0"); assert(a != b, "a != b with a:'1.0.0', b:'2.0.0' failed"); + a = Version.masterBranch; b = Version("~BRANCH"); assert(a != b, "a != b with a:MASTER, b:'~branch' failed"); @@ -829,4 +966,22 @@ assertThrown(a == b, "Failed: UNKNOWN == UNKNOWN"); assert(Version("1.0.0+a") == Version("1.0.0+b")); + + assert(Version("73535568b79a0b124bc1653002637a830ce0fcb8").isSCM); +} + +/// Determines whether the given string is a Git hash. +bool isGitHash(string hash) @nogc nothrow pure @safe +{ + import std.ascii : isHexDigit; + import std.utf : byCodeUnit; + + return hash.length >= 7 && hash.length <= 40 && hash.byCodeUnit.all!isHexDigit; +} + +@nogc nothrow pure @safe unittest { + assert(isGitHash("73535568b79a0b124bc1653002637a830ce0fcb8")); + assert(!isGitHash("735")); + assert(!isGitHash("73535568b79a0b124bc1-53002637a830ce0fcb8")); + assert(!isGitHash("73535568b79a0b124bg1")); } diff --git a/source/dub/dub.d b/source/dub/dub.d index 4f5de93..39a5b71 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -481,6 +481,9 @@ if (!path.absolute) path = this.rootPath ~ path; try if (m_packageManager.getOrLoadPackage(path)) continue; catch (Exception e) { logDebug("Failed to load path based selection: %s", e.toString().sanitize); } + } else if (!dep.repository.empty) { + if (m_packageManager.loadSCMPackage(getBasePackageName(p), dep)) + continue; } else { if (m_packageManager.getPackage(p, dep.version_)) continue; foreach (ps; m_packageSuppliers) { @@ -511,7 +514,7 @@ string rootbasename = getBasePackageName(m_project.rootPackage.name); foreach (p, ver; versions) { - if (!ver.path.empty) continue; + if (!ver.path.empty || !ver.repository.empty) continue; auto basename = getBasePackageName(p); if (basename == rootbasename) continue; @@ -523,7 +526,7 @@ continue; } auto sver = m_project.selections.getSelectedVersion(basename); - if (!sver.path.empty) continue; + if (!sver.path.empty || !sver.repository.empty) continue; if (ver.version_ <= sver.version_) continue; logInfo("Package %s would be upgraded from %s to %s.", basename, sver, ver); @@ -543,6 +546,8 @@ logDebug("Failed to load path based selection: %s", e.toString().sanitize); continue; } + } else if (!ver.repository.empty) { + pack = m_packageManager.loadSCMPackage(p, ver); } else { pack = m_packageManager.getBestPackage(p, ver); if (pack && m_packageManager.isManagedPackage(pack) @@ -559,8 +564,11 @@ fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none; if (!pack) fetch(p, ver, defaultPlacementLocation, fetchOpts, "getting selected version"); if ((options & UpgradeOptions.select) && p != m_project.rootPackage.name) { - if (ver.path.empty) m_project.selections.selectVersion(p, ver.version_); - else { + if (!ver.repository.empty) { + m_project.selections.selectVersionWithRepository(p, ver.repository, ver.versionSpec); + } else if (ver.path.empty) { + m_project.selections.selectVersion(p, ver.version_); + } else { NativePath relpath = ver.path; if (relpath.absolute) relpath = relpath.relativeTo(m_project.rootPackage.path); m_project.selections.selectVersion(p, relpath); @@ -1553,7 +1561,10 @@ protected override Dependency[] getSpecificConfigs(string pack, TreeNodes nodes) { - if (!nodes.configs.path.empty && getPackage(pack, nodes.configs)) return [nodes.configs]; + if (!nodes.configs.path.empty || !nodes.configs.repository.empty) { + if (getPackage(pack, nodes.configs)) return [nodes.configs]; + else return null; + } else return null; } @@ -1681,7 +1692,10 @@ if (basename == m_rootPackage.basePackage.name) return m_rootPackage.basePackage; - if (!dep.path.empty) { + if (!dep.repository.empty) { + auto ret = m_dub.packageManager.loadSCMPackage(name, dep); + return ret !is null && dep.matches(ret.version_) ? ret : null; + } else if (!dep.path.empty) { try { auto ret = m_dub.packageManager.getOrLoadPackage(dep.path); if (dep.matches(ret.version_)) return ret; diff --git a/source/dub/internal/git.d b/source/dub/internal/git.d index 50be963..d1e52d6 100644 --- a/source/dub/internal/git.d +++ b/source/dub/internal/git.d @@ -101,3 +101,41 @@ return null; } + +/** Clones a repository into a new directory. + + Params: + remote = The (possibly remote) repository to clone from + reference = The branch to check out after cloning + destination = Repository destination directory + + Returns: + Whether the cloning succeeded. +*/ +bool cloneRepository(string remote, string reference, string destination) +{ + import std.process : Pid, spawnProcess, wait; + + Pid command; + + if (!exists(destination)) { + string[] args = ["git", "clone", "--no-checkout"]; + if (getLogLevel > LogLevel.diagnostic) args ~= "-q"; + + command = spawnProcess(args~[remote, destination]); + if (wait(command) != 0) { + return false; + } + } + + string[] args = ["git", "-C", destination, "checkout", "--detach"]; + if (getLogLevel > LogLevel.diagnostic) args ~= "-q"; + command = spawnProcess(args~[reference]); + + if (wait(command) != 0) { + rmdirRecurse(destination); + return false; + } + + return true; +} diff --git a/source/dub/packagemanager.d b/source/dub/packagemanager.d index a4a23f2..be15f15 100644 --- a/source/dub/packagemanager.d +++ b/source/dub/packagemanager.d @@ -254,6 +254,62 @@ return pack; } + /** For a given SCM repository, returns the corresponding package. + + An SCM repository is provided as its remote URL, the repository is cloned + and in the dependency speicfied commit is checked out. + + If the target directory already exists, just returns the package + without cloning. + + Params: + name = Package name + dependency = Dependency that contains the repository URL and a specific commit + + Returns: + The package loaded from the given SCM repository or null if the + package couldn't be loaded. + */ + Package loadSCMPackage(string name, Dependency dependency) + in { assert(!dependency.repository.empty); } + body { + Package pack; + + with (dependency.repository) final switch (kind) + { + case Kind.git: + pack = loadGitPackage(name, dependency.versionSpec, dependency.repository.remote); + } + if (pack !is null) { + addPackages(m_temporaryPackages, pack); + } + return pack; + } + + private Package loadGitPackage(string name, string versionSpec, string remote) + { + import dub.internal.git : cloneRepository; + + if (!versionSpec.startsWith("~") && !versionSpec.isGitHash) { + return null; + } + + string gitReference = versionSpec.chompPrefix("~"); + const destination = m_repositories[LocalPackageType.user].packagePath ~ + NativePath(name ~ "-" ~ gitReference) ~ (name~"/"); + + foreach (p; getPackageIterator(name)) { + if (p.path == destination) { + return p; + } + } + + if (!cloneRepository(remote, gitReference, destination.toNativeString())) { + return null; + } + + return Package.load(destination); + } /** Searches for the latest version of a package matching the given dependency. */ diff --git a/source/dub/project.d b/source/dub/project.d index f02f47e..edfd20b 100644 --- a/source/dub/project.d +++ b/source/dub/project.d @@ -256,7 +256,7 @@ enforce(!m_rootPackage.name.canFind(' '), "Aborting due to the package name containing spaces."); foreach (d; m_rootPackage.getAllDependencies()) - if (d.spec.isExactVersion && d.spec.version_.isBranch) { + if (d.spec.isExactVersion && d.spec.version_.isBranch && d.spec.repository.empty) { logWarn("WARNING: A deprecated branch based version specification is used " ~ "for the dependency %s. Please use numbered versions instead. Also " ~ "note that you can still use the %s file to override a certain " @@ -341,6 +341,10 @@ // need to be satisfied bool is_desired = !vspec.optional || m_selections.hasSelectedVersion(basename) || (vspec.default_ && m_selections.bare); + Package resolveSubPackage(Package p, in bool silentFail) { + return subname.length ? m_packageManager.getSubPackage(p, subname, silentFail) : p; + } + if (dep.name == m_rootPackage.basePackage.name) { vspec = Dependency(m_rootPackage.version_); p = m_rootPackage.basePackage; @@ -354,24 +358,32 @@ } } else if (m_selections.hasSelectedVersion(basename)) { vspec = m_selections.getSelectedVersion(basename); - if (vspec.path.empty) p = m_packageManager.getBestPackage(dep.name, vspec); - else { + if (!vspec.path.empty) { auto path = vspec.path; if (!path.absolute) path = m_rootPackage.path ~ path; p = m_packageManager.getOrLoadPackage(path, NativePath.init, true); - if (subname.length) p = m_packageManager.getSubPackage(p, subname, true); + p = resolveSubPackage(p, true); + } else if (!vspec.repository.empty) { + p = m_packageManager.loadSCMPackage(basename, vspec); + p = resolveSubPackage(p, true); + } else { + p = m_packageManager.getBestPackage(dep.name, vspec); } } else if (m_dependencies.canFind!(d => getBasePackageName(d.name) == basename)) { auto idx = m_dependencies.countUntil!(d => getBasePackageName(d.name) == basename); auto bp = m_dependencies[idx].basePackage; vspec = Dependency(bp.path); - if (subname.length) p = m_packageManager.getSubPackage(bp, subname, false); - else p = bp; + p = resolveSubPackage(bp, false); } else { logDiagnostic("%sVersion selection for dependency %s (%s) of %s is missing.", indent, basename, dep.name, pack.name); } + if (!p && !vspec.repository.empty) { + p = m_packageManager.loadSCMPackage(basename, vspec); + resolveSubPackage(p, false); + } + if (!p && !vspec.path.empty) { NativePath path = vspec.path; if (!path.absolute) path = pack.path ~ path; @@ -381,7 +393,7 @@ logWarn("%sSub package %s must be referenced using the path to it's parent package.", indent, dep.name); p = p.parentPackage; } - if (subname.length) p = m_packageManager.getSubPackage(p, subname, false); + p = resolveSubPackage(p, false); enforce(p.name == dep.name, format("Path based dependency %s is referenced with a wrong name: %s vs. %s", path.toNativeString(), dep.name, p.name)); @@ -1542,6 +1554,18 @@ m_dirty = true; } + /// Selects a certain Git reference for a specific package. + void selectVersionWithRepository(string package_id, Repository repository, string spec) + { + const dependency = Dependency(repository, spec); + if (auto ps = package_id in m_selections) { + if (ps.dep == dependency) + return; + } + m_selections[package_id] = Selected(dependency); + m_dirty = true; + } + /// Removes the selection for a particular package. void deselectVersion(string package_id) { @@ -1604,7 +1628,12 @@ static Json dependencyToJson(Dependency d) { - if (d.path.empty) return Json(d.version_.toString()); + if (!d.repository.empty) { + return serializeToJson([ + "version": d.version_.toString(), + "repository": d.repository.toString, + ]); + } else if (d.path.empty) return Json(d.version_.toString()); else return serializeToJson(["path": d.path.toString()]); } @@ -1612,9 +1641,12 @@ { if (j.type == Json.Type.string) return Dependency(Version(j.get!string)); - else if (j.type == Json.Type.object) + else if (j.type == Json.Type.object && "path" in j) return Dependency(NativePath(j["path"].get!string)); - else throw new Exception(format("Unexpected type for dependency: %s", j.type)); + else if (j.type == Json.Type.object && "repository" in j) + return Dependency(Repository(j["repository"].get!string), + enforce("version" in j, "Expected \"version\" field in repository version object").get!string); + else throw new Exception(format("Unexpected type for dependency: %s", j)); } Json serialize() diff --git a/source/dub/recipe/sdl.d b/source/dub/recipe/sdl.d index 7f3ad45..d13d4c8 100644 --- a/source/dub/recipe/sdl.d +++ b/source/dub/recipe/sdl.d @@ -187,6 +187,11 @@ logDiagnostic("Ignoring version specification (%s) for path based dependency %s", attrs["version"][0].value.get!string, attrs["path"][0].value.get!string); dep.versionSpec = "*"; dep.path = NativePath(attrs["path"][0].value.get!string); + } else if ("repository" in attrs) { + enforceSDL("version" in attrs, "Missing version specification.", t); + + dep.repository = Repository(attrs["repository"][0].value.get!string); + dep.versionSpec = attrs["version"][0].value.get!string; } else { enforceSDL("version" in attrs, "Missing version specification.", t); dep.versionSpec = attrs["version"][0].value.get!string; @@ -239,6 +244,7 @@ foreach (pack, d; bs.dependencies) { Attribute[] attribs; + if (!d.repository.empty) attribs ~= new Attribute(null, "repository", Value(d.repository.toString())); if (!d.path.empty) attribs ~= new Attribute(null, "path", Value(d.path.toString())); else attribs ~= new Attribute(null, "version", Value(d.versionSpec)); if (d.optional) attribs ~= new Attribute(null, "optional", Value(true)); @@ -586,3 +592,28 @@ parseSDL(rec, sdl, null, "testfile"); assert("" in rec.buildSettings.sourcePaths); } + +unittest { + auto sdl = +`name "test" +dependency "package" repository="git+https://some.url" version="12345678" +`; + PackageRecipe rec; + parseSDL(rec, sdl, null, "testfile"); + auto dependency = rec.buildSettings.dependencies["package"]; + assert(!dependency.repository.empty); + assert(dependency.versionSpec == "12345678"); +} + +unittest { + PackageRecipe p; + p.name = "test"; + + auto repository = Repository("git+https://some.url"); + p.buildSettings.dependencies["package"] = Dependency(repository, "12345678"); + auto sdl = toSDL(p).toSDLDocument(); + assert(sdl == +`name "test" +dependency "package" repository="git+https://some.url" version="12345678" +`); +} diff --git a/test/git-dependency/dub.json b/test/git-dependency/dub.json new file mode 100644 index 0000000..ace7c64 --- /dev/null +++ b/test/git-dependency/dub.json @@ -0,0 +1,9 @@ +{ + "name": "git-dependency", + "dependencies": { + "gitcompatibledubpackage": { + "repository": "git+https://github.com/dlang-community/gitcompatibledubpackage.git", + "version": "ccb31bf6a655437176ec02e04c2305a8c7c90d67" + } + } +} diff --git a/test/git-dependency/src/app.d b/test/git-dependency/src/app.d new file mode 100644 index 0000000..29751b2 --- /dev/null +++ b/test/git-dependency/src/app.d @@ -0,0 +1,6 @@ +import gitcompatibledubpackage.subdir.file; + +void main() +{ + assert(!hasTheWorldExploded()); +}