diff --git a/source/dub/commandline.d b/source/dub/commandline.d index ec826d5..91b0b32 100644 --- a/source/dub/commandline.d +++ b/source/dub/commandline.d @@ -224,7 +224,20 @@ dub.defaultPlacementLocation = options.placementLocation; } else { // initialize DUB - auto package_suppliers = options.registry_urls.map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url))).array; + auto package_suppliers = options.registry_urls + .map!((url) { + // Allow to specify fallback mirrors as space separated urls. Undocumented as we + // should simply retry over all registries instead of using a special + // FallbackPackageSupplier. + auto urls = url.splitter(' '); + PackageSupplier ps = new RegistryPackageSupplier(URL(urls.front)); + urls.popFront; + if (!urls.empty) + ps = new FallbackPackageSupplier(ps, + urls.map!(u => cast(PackageSupplier) new RegistryPackageSupplier(URL(u))).array); + return ps; + }) + .array; dub = new Dub(options.root_path, package_suppliers, options.skipRegistry); dub.dryRun = options.annotate; dub.defaultPlacementLocation = options.placementLocation; diff --git a/source/dub/dub.d b/source/dub/dub.d index 57c91d6..47bf48b 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -791,12 +791,26 @@ return m_packageManager.getPackage(packageId, ver, dstpath); } - auto path = getTempFile(packageId, ".zip"); - supplier.fetchPackage(path, packageId, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail? - scope(exit) std.file.remove(path.toNativeString()); + // repeat download on corrupted zips, see #1336 + foreach_reverse (i; 0..3) + { + import std.zip : ZipException; - logDiagnostic("Placing to %s...", placement.toNativeString()); - return m_packageManager.storeFetchedPackage(path, pinfo, dstpath); + auto path = getTempFile(packageId, ".zip"); + supplier.fetchPackage(path, packageId, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail? + scope(exit) std.file.remove(path.toNativeString()); + logDiagnostic("Placing to %s...", placement.toNativeString()); + + try { + return m_packageManager.storeFetchedPackage(path, pinfo, dstpath); + } catch (ZipException e) { + logInfo("Failed to extract zip archive for %s %s...", packageId, ver); + // rethrow the exception at the end of the loop + if (i == 0) + throw e; + } + } + assert(0, "Should throw a ZipException instead."); } /** Removes a specific locally cached package. diff --git a/source/dub/internal/utils.d b/source/dub/internal/utils.d index d85e262..22e6244 100644 --- a/source/dub/internal/utils.d +++ b/source/dub/internal/utils.d @@ -265,7 +265,15 @@ } } else + { std.net.curl.download(url, filename, conn); + // workaround https://issues.dlang.org/show_bug.cgi?id=18318 + auto sl = conn.statusLine; + logDebug("Download %s %s", url, sl); + if (sl.code / 100 != 2) + throw new HTTPStatusException(sl.code, + "Downloading %s failed with %d (%s).".format(url, sl.code, sl.reason)); + } } else version (Have_vibe_d_http) { import vibe.inet.urltransfer; vibe.inet.urltransfer.download(url, filename); diff --git a/source/dub/version_.d b/source/dub/version_.d index 186a95d..10c03cf 100644 --- a/source/dub/version_.d +++ b/source/dub/version_.d @@ -1,2 +1,2 @@ module dub.version_; -enum dubVersion = "v1.7.1"; +enum dubVersion = "v1.7.2-beta.1"; diff --git a/test/.gitignore b/test/.gitignore index adf9c15..77ea5b9 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -11,3 +11,5 @@ custom-unittest/custom-unittest path-subpackage-ref/test subpackage-ref/test + +/test_registry diff --git a/test/fetchzip.sh b/test/fetchzip.sh new file mode 100755 index 0000000..b0d34d2 --- /dev/null +++ b/test/fetchzip.sh @@ -0,0 +1,59 @@ +#!/bin/bash +DIR=$(dirname "${BASH_SOURCE[0]}") + +. "$DIR"/common.sh + +PORT=$(($$ + 1024)) # PID + 1024 + +dub remove gitcompatibledubpackage --non-interactive --version=* 2>/dev/null || true + +"$DUB" build --single "$DIR"/test_registry.d +"$DIR"/test_registry --folder="$DIR/issue1336-registry" --port=$PORT & +PID=$! +sleep 0.2 +trap 'kill $PID 2>/dev/null || true' exit + +echo "Trying to download gitcompatibledubpackage (1.0.4)" +timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.4 --skip-registry=all --registry=http://localhost:$PORT +if [ $? -eq 124 ]; then + die 'Fetching from responsive registry should not time-out.' +fi +dub remove gitcompatibledubpackage --non-interactive --version=1.0.4 + +echo "Downloads should be retried when the zip is corrupted - gitcompatibledubpackage (1.0.3)" +zipOut=$(! timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.3 --skip-registry=all --registry=http://localhost:$PORT 2>&1) +rc=$? + +if ! zipCount=$(grep -Fc 'Failed to extract zip archive' <<<"$zipOut") || [ "$zipCount" -lt 3 ] ; then + echo '========== +Output was ==========' >&2 + echo "$zipOut" >&2 + echo '========== -Output was ==========' >&2 + die 'DUB should have tried to download the zip archive multiple times.' +elif [ $rc -eq 124 ]; then + die 'DUB timed out unexpectedly.' +fi +if dub remove gitcompatibledubpackage --non-interactive --version=* 2>/dev/null; then + die 'DUB should not have installed a broken package.' +fi + +echo "HTTP status errors on downloads should be retried - gitcompatibledubpackage (1.0.2)" +retryOut=$(! timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.2 --skip-registry=all --registry=http://localhost:$PORT --vverbose 2>&1) +rc=$? +if ! retryCount=$(echo "$retryOut" | grep -Fc 'Bad Gateway') || [ "$retryCount" -lt 3 ] ; then + echo '========== +Output was ==========' >&2 + echo "$retryOut" >&2 + echo '========== -Output was ==========' >&2 + die "DUB should have retried download on server error multiple times, but only tried $retryCount times." +elif [ $rc -eq 124 ]; then + die 'DUB timed out unexpectedly.' +fi +if dub remove gitcompatibledubpackage --non-interactive --version=* 2>/dev/null; then + die 'DUB should not have installed a package.' +fi + +echo "HTTP status errors on downloads should retry with fallback mirror - gitcompatibledubpackage (1.0.2)" +timeout 1s "$DUB" fetch gitcompatibledubpackage --version=1.0.2 --skip-registry=all --registry="http://localhost:$PORT http://localhost:$PORT/fallback" +if [ $? -eq 124 ]; then + die 'Fetching from responsive registry should not time-out.' +fi +dub remove gitcompatibledubpackage --non-interactive --version=1.0.2 diff --git a/test/fetchzip.sh.min_frontend b/test/fetchzip.sh.min_frontend new file mode 100644 index 0000000..bb0a2e1 --- /dev/null +++ b/test/fetchzip.sh.min_frontend @@ -0,0 +1 @@ +2.076 diff --git a/test/issue1336-registry/.gitignore b/test/issue1336-registry/.gitignore new file mode 100644 index 0000000..86e1bdf --- /dev/null +++ b/test/issue1336-registry/.gitignore @@ -0,0 +1,14 @@ +.dub +docs.json +__dummy.html +docs/ +issue1336-registry.so +issue1336-registry.dylib +issue1336-registry.dll +issue1336-registry.a +issue1336-registry.lib +issue1336-registry-test-* +*.exe +*.o +*.obj +*.lst diff --git a/test/issue1336-registry/.no_build b/test/issue1336-registry/.no_build new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/test/issue1336-registry/.no_build diff --git a/test/issue1336-registry/packages/gitcompatibledubpackage.json b/test/issue1336-registry/packages/gitcompatibledubpackage.json new file mode 100644 index 0000000..6f0d641 --- /dev/null +++ b/test/issue1336-registry/packages/gitcompatibledubpackage.json @@ -0,0 +1 @@ +{"repository":{"project":"gitcompatibledubpackage","owner":"dlang-community","kind":"github"},"dateAdded":"2017-02-11T21:00:16","name":"gitcompatibledubpackage","categories":[],"owner":"5148973d2179ddb20b0002fd","id":"589f6d501c18646d0ee49f12","versions":[{"packageDescriptionFile":"dub.sdl","configurations":[{"name":"exe","targetType":"executable","mainSourceFile":"hello.d"},{"excludedSourceFiles":["hello.d"],"name":"lib","targetType":"library"}],"commitID":"9e3972be4c63790c32257220f40c0af7dc41bec5","importPaths":[".."],"name":"gitcompatibledubpackage","version":"~master","license":"public domain","readmeFile":"/README.md","date":"2017-09-18T18:18:35Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","sourcePaths":["."]},{"packageDescriptionFile":"dub.json","commitID":"b62466d32dd6bbb0d45f9a73c91142205d1048e5","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.1","license":"public domain","readmeFile":"/README.md","date":"2015-10-31T11:09:14Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite."},{"packageDescriptionFile":"dub.json","commitID":"cef89b5513a140b1e3417809f59d1957ddaad837","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.2","license":"public domain","readmeFile":"/README.md","date":"2015-10-31T11:23:53Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","sourcePaths":["."]},{"packageDescriptionFile":"dub.json","commitID":"d36e49d31b26ea76e1670e9d5cd89aaac40fde75","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.3","targetType":"executable","license":"public domain","readmeFile":"/README.md","date":"2017-04-05T22:12:48Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","mainSourceFile":"hello.d","sourcePaths":["."]},{"packageDescriptionFile":"dub.sdl","configurations":[{"name":"exe","targetType":"executable","mainSourceFile":"hello.d"},{"excludedSourceFiles":["hello.d"],"name":"lib","targetType":"library"}],"commitID":"9e3972be4c63790c32257220f40c0af7dc41bec5","importPaths":[".."],"name":"gitcompatibledubpackage","version":"1.0.4","license":"public domain","readmeFile":"/README.md","date":"2017-09-18T18:18:35Z","description":"Example of a DUB package also usable as git submodule. For DUB test suite.","sourcePaths":["."]}]} \ No newline at end of file diff --git a/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.2.zip b/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.2.zip new file mode 120000 index 0000000..cd46966 --- /dev/null +++ b/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.2.zip @@ -0,0 +1 @@ +1.0.4.zip \ No newline at end of file diff --git a/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.3.zip b/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.3.zip new file mode 100644 index 0000000..28212ee --- /dev/null +++ b/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.3.zip @@ -0,0 +1 @@ +BROKEN ZIP diff --git a/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.4.zip b/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.4.zip new file mode 100644 index 0000000..d16cdc9 --- /dev/null +++ b/test/issue1336-registry/packages/gitcompatibledubpackage/1.0.4.zip Binary files differ diff --git a/test/test_registry.d b/test/test_registry.d new file mode 100644 index 0000000..dfeb358 --- /dev/null +++ b/test/test_registry.d @@ -0,0 +1,23 @@ +/+dub.sdl: +dependency "vibe-d" version="~>0.8.3-alpha.1" +versions "VibeNoSSL" ++/ + +void main(string[] args) +{ + import std.conv, vibe.d; + immutable folder = readRequiredOption!string("folder", "Folder to service files from."); + immutable port = readRequiredOption!uint("port", "Port to use"); + auto router = new URLRouter; + router.get("stop", (HTTPServerRequest req, HTTPServerResponse res){ + res.writeVoidBody; + exitEventLoop(); + }); + router.get("/packages/gitcompatibledubpackage/1.0.2.zip", (req, res) { + res.writeBody("", HTTPStatus.badGateway); + }); + router.get("*", folder.serveStaticFiles); + router.get("/fallback/*", folder.serveStaticFiles(new HTTPFileServerSettings("/fallback"))); + listenHTTP(text(":", port), router); + runApplication(); +} diff --git a/travis-ci.sh b/travis-ci.sh index 8fc4255..c7782f2 100755 --- a/travis-ci.sh +++ b/travis-ci.sh @@ -19,11 +19,13 @@ DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh deactivate git clean -dxf -- test - source $(~/dlang/install.sh ldc --activate) + export FRONTEND=2.077 + source $(~/dlang/install.sh ldc-1.7.0 --activate) DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh deactivate git clean -dxf -- test - source $(~/dlang/install.sh gdc --activate) + export FRONTEND=2.068 + source $(~/dlang/install.sh gdc-4.8.5 --activate) DUB=`pwd`/bin/dub DC=${DC} test/run-unittest.sh deactivate else