diff --git a/source/dub/commandline.d b/source/dub/commandline.d index f3825c8..94d55b1 100644 --- a/source/dub/commandline.d +++ b/source/dub/commandline.d @@ -230,11 +230,11 @@ // should simply retry over all registries instead of using a special // FallbackPackageSupplier. auto urls = url.splitter(' '); - PackageSupplier ps = new RegistryPackageSupplier(URL(urls.front)); + PackageSupplier ps = getRegistryPackageSupplier(urls.front); urls.popFront; if (!urls.empty) ps = new FallbackPackageSupplier(ps, - urls.map!(u => cast(PackageSupplier) new RegistryPackageSupplier(URL(u))).array); + urls.map!(u => getRegistryPackageSupplier(u)).array); return ps; }) .array; diff --git a/source/dub/dub.d b/source/dub/dub.d index 47bf48b..d088c5a 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -87,6 +87,34 @@ ]; } +/** Returns a registry package supplier according to protocol. + + Allowed protocols are dub+http(s):// and maven+http(s)://. +*/ +PackageSupplier getRegistryPackageSupplier(string url) +{ + switch (url.startsWith("dub+", "mvn+")) + { + case 1: + return new RegistryPackageSupplier(URL(url[4..$])); + case 2: + return new MavenRegistryPackageSupplier(URL(url[4..$])); + default: + return new RegistryPackageSupplier(URL(url)); + } +} + +unittest +{ + auto dubRegistryPackageSupplier = getRegistryPackageSupplier("dub+https://code.dlang.org"); + assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org")); + + dubRegistryPackageSupplier = getRegistryPackageSupplier("https://code.dlang.org"); + assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org")); + + auto mavenRegistryPackageSupplier = getRegistryPackageSupplier("mvn+http://localhost:8040/maven/libs-release/dubpackages"); + assert(mavenRegistryPackageSupplier.description.canFind(" http://localhost:8040/maven/libs-release/dubpackages")); +} /** Provides a high-level entry point for DUB's functionality. @@ -145,14 +173,14 @@ { ps ~= environment.get("DUB_REGISTRY", null) .splitter(";") - .map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url))) + .map!(url => getRegistryPackageSupplier(url)) .array; } if (skip_registry < SkipPackageSuppliers.configured) { ps ~= m_config.registryURLs - .map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url))) + .map!(url => getRegistryPackageSupplier(url)) .array; } diff --git a/source/dub/packagesupplier.d b/source/dub/packagesupplier.d index 44ddbcf..eb2806c 100644 --- a/source/dub/packagesupplier.d +++ b/source/dub/packagesupplier.d @@ -186,7 +186,8 @@ void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release) { import std.array : replace; - Json best = getBestPackage(packageId, dep, pre_release); + auto md = getMetadata(packageId); + Json best = getBestPackage(md, packageId, dep, pre_release); if (best.type == Json.Type.null_) return; auto vers = best["version"].get!string; @@ -210,7 +211,8 @@ Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release) { - return getBestPackage(packageId, dep, pre_release); + auto md = getMetadata(packageId); + return getBestPackage(md, packageId, dep, pre_release); } private Json getMetadata(string packageId) @@ -266,27 +268,129 @@ .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string)) .array; } +} - private Json getBestPackage(string packageId, Dependency dep, bool pre_release) +/** + Maven repository based package supplier. + + This package supplier connects to a maven repository + to search for available packages. +*/ +class MavenRegistryPackageSupplier : PackageSupplier { + private { + URL m_mavenUrl; + struct CacheEntry { Json data; SysTime cacheTime; } + CacheEntry[string] m_metadataCache; + Duration m_maxCacheTime; + } + + this(URL mavenUrl) { - Json md = getMetadata(packageId); + m_mavenUrl = mavenUrl; + m_maxCacheTime = 24.hours(); + } + + override @property string description() { return "maven repository at "~m_mavenUrl.toString(); } + + Version[] getVersions(string package_id) + { + auto md = getMetadata(package_id); if (md.type == Json.Type.null_) - return md; - Json best = null; - Version bestver; + return null; + Version[] ret; foreach (json; md["versions"]) { - auto cur = Version(cast(string)json["version"]); - if (!dep.matches(cur)) continue; - if (best == null) best = json; - else if (pre_release) { - if (cur > bestver) best = json; - } else if (bestver.isPreRelease) { - if (!cur.isPreRelease || cur > bestver) best = json; - } else if (!cur.isPreRelease && cur > bestver) best = json; - bestver = Version(cast(string)best["version"]); + auto cur = Version(json["version"].get!string); + ret ~= cur; } - enforce(best != null, "No package candidate found for "~packageId~" "~dep.toString()); - return best; + ret.sort(); + return ret; + } + + void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release) + { + auto md = getMetadata(packageId); + Json best = getBestPackage(md, packageId, dep, pre_release); + if (best.type == Json.Type.null_) + return; + auto vers = best["version"].get!string; + auto url = m_mavenUrl~NativePath("%s/%s/%s-%s.zip".format(packageId, vers, packageId, vers)); + logDiagnostic("Downloading from '%s'", url); + foreach(i; 0..3) { + try{ + download(url, path); + return; + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download package %s from %s (Attempt %s of 3)", packageId, url, i + 1); + continue; + } + } + } + throw new Exception("Failed to download package %s from %s".format(packageId, url)); + } + + Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release) + { + auto md = getMetadata(packageId); + return getBestPackage(md, packageId, dep, pre_release); + } + + private Json getMetadata(string packageId) + { + import std.xml; + + auto now = Clock.currTime(UTC()); + if (auto pentry = packageId in m_metadataCache) { + if (pentry.cacheTime + m_maxCacheTime > now) + return pentry.data; + m_metadataCache.remove(packageId); + } + + auto url = m_mavenUrl~NativePath(packageId~"/maven-metadata.xml"); + + logDebug("Downloading maven metadata for %s", packageId); + logDebug("Getting from %s", url); + + string xmlData; + foreach(i; 0..3) { + try { + xmlData = cast(string)download(url); + break; + } + catch (HTTPStatusException e) + { + if (e.status == 404) { + logDebug("Maven metadata %s not found at %s (404): %s", packageId, description, e.msg); + return Json(null); + } + else { + logDebug("Error getting maven metadata for %s at %s (attempt %s of 3): %s", packageId, description, i + 1, e.msg); + if (i == 2) + throw e; + continue; + } + } + } + + auto json = Json(["name": Json(packageId), "versions": Json.emptyArray]); + auto xml = new DocumentParser(xmlData); + + xml.onStartTag["versions"] = (ElementParser xml) { + xml.onEndTag["version"] = (in Element e) { + json["versions"] ~= serializeToJson(["name": packageId, "version": e.text]); + }; + xml.parse(); + }; + xml.parse(); + m_metadataCache[packageId] = CacheEntry(json, now); + return json; + } + + SearchResult[] searchPackages(string query) + { + return []; } } @@ -342,3 +446,24 @@ } private enum PackagesPath = "packages"; + +private Json getBestPackage(Json metadata, string packageId, Dependency dep, bool pre_release) +{ + if (metadata.type == Json.Type.null_) + return metadata; + Json best = null; + Version bestver; + foreach (json; metadata["versions"]) { + auto cur = Version(json["version"].get!string); + if (!dep.matches(cur)) continue; + if (best == null) best = json; + else if (pre_release) { + if (cur > bestver) best = json; + } else if (bestver.isPreRelease) { + if (!cur.isPreRelease || cur > bestver) best = json; + } else if (!cur.isPreRelease && cur > bestver) best = json; + bestver = Version(cast(string)best["version"]); + } + enforce(best != null, "No package candidate found for "~packageId~" "~dep.toString()); + return best; +} diff --git a/test/issue1416-maven-repo-pkg-supplier.sh b/test/issue1416-maven-repo-pkg-supplier.sh new file mode 100755 index 0000000..4f106b4 --- /dev/null +++ b/test/issue1416-maven-repo-pkg-supplier.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +DIR=$(dirname "${BASH_SOURCE[0]}") + +. "$DIR"/common.sh + +PORT=$(($$ + 1024)) # PID + 1024 + +dub remove maven-dubpackage --non-interactive --version=* 2>/dev/null || true + +"$DUB" build --single "$DIR"/test_registry.d +"$DIR"/test_registry --folder="$DIR/issue1416-maven-repo-pkg-supplier" --port=$PORT & +PID=$! +sleep 1 +trap 'kill $PID 2>/dev/null || true' exit + +echo "Trying to download maven-dubpackage (1.0.5)" +"$DUB" fetch maven-dubpackage --version=1.0.5 --skip-registry=all --registry=mvn+http://localhost:$PORT/maven/release/dubpackages + +if ! dub remove maven-dubpackage --non-interactive --version=1.0.5 2>/dev/null; then + die 'DUB did not install package from maven registry.' +fi + +echo "Trying to download maven-dubpackage (latest)" +"$DUB" fetch maven-dubpackage --skip-registry=all --registry=mvn+http://localhost:$PORT/maven/release/dubpackages + +if ! dub remove maven-dubpackage --non-interactive --version=1.0.6 2>/dev/null; then + die 'DUB did not install latest package from maven registry.' +fi \ No newline at end of file diff --git a/test/issue1416-maven-repo-pkg-supplier.sh.min_frontend b/test/issue1416-maven-repo-pkg-supplier.sh.min_frontend new file mode 100644 index 0000000..bb0a2e1 --- /dev/null +++ b/test/issue1416-maven-repo-pkg-supplier.sh.min_frontend @@ -0,0 +1 @@ +2.076 diff --git a/test/issue1416-maven-repo-pkg-supplier/.gitignore b/test/issue1416-maven-repo-pkg-supplier/.gitignore new file mode 100644 index 0000000..304e955 --- /dev/null +++ b/test/issue1416-maven-repo-pkg-supplier/.gitignore @@ -0,0 +1,8 @@ +.dub +docs.json +__dummy.html +docs/ +*.exe +*.o +*.obj +*.lst diff --git a/test/issue1416-maven-repo-pkg-supplier/.no_build b/test/issue1416-maven-repo-pkg-supplier/.no_build new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/issue1416-maven-repo-pkg-supplier/.no_build diff --git a/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/1.0.5/maven-dubpackage-1.0.5.zip b/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/1.0.5/maven-dubpackage-1.0.5.zip new file mode 100644 index 0000000..c3e7d58 --- /dev/null +++ b/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/1.0.5/maven-dubpackage-1.0.5.zip Binary files differ diff --git a/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/1.0.6/maven-dubpackage-1.0.6.zip b/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/1.0.6/maven-dubpackage-1.0.6.zip new file mode 100644 index 0000000..333d920 --- /dev/null +++ b/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/1.0.6/maven-dubpackage-1.0.6.zip Binary files differ diff --git a/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/maven-metadata.xml b/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/maven-metadata.xml new file mode 100644 index 0000000..7b171d5 --- /dev/null +++ b/test/issue1416-maven-repo-pkg-supplier/maven/release/dubpackages/maven-dubpackage/maven-metadata.xml @@ -0,0 +1,14 @@ + + + dubpackages + maven-dubpackage + + 1.0.6 + 1.0.6 + + 1.0.5 + 1.0.6 + + 20180317184845 + + \ No newline at end of file