diff --git a/.gitignore b/.gitignore index 2370208..47dcf7d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ # Ignore build files. /bin/dub +/bin/__test__library-nonet__ /bin/__test__library__ /bin/dub-test-library /bin/libdub.a diff --git a/dub.sdl b/dub.sdl index f45bc24..2ae4255 100644 --- a/dub.sdl +++ b/dub.sdl @@ -10,7 +10,7 @@ configuration "application" { targetType "executable" mainSourceFile "source/app.d" - versions "DubApplication" + versions "DubUseCurl" "DubApplication" // Uncomment to get rich output about the file parsing and json <-> YAML // integrity checks //debugVersions "ConfigFillerDebug" @@ -20,4 +20,17 @@ targetType "library" excludedSourceFiles "source/app.d" copyFiles "bin/libcurl.dll" "bin/libeay32.dll" "bin/ssleay32.dll" platform="windows" + versions "DubUseCurl" +} + +configuration "library-nonet" { + dependency "vibe-d:http" version=">=0.9.0 <0.10.0" optional=true + targetType "library" + excludedSourceFiles "source/app.d" +} + +configuration "dynamic-library-nonet" { + dependency "vibe-d:http" version=">=0.9.0 <0.10.0" optional=true + targetType "dynamicLibrary" + excludedSourceFiles "source/app.d" } diff --git a/scripts/ci/travis.sh b/scripts/ci/travis.sh index b72fb97..a811e75 100755 --- a/scripts/ci/travis.sh +++ b/scripts/ci/travis.sh @@ -4,6 +4,7 @@ vibe_ver=$(jq -r '.versions | .["vibe-d"]' < dub.selections.json) dub fetch vibe-d@$vibe_ver # get optional dependency +dub test --compiler=${DC} -c library-nonet export DMD="$(command -v $DMD)" @@ -19,6 +20,7 @@ } if [ "$COVERAGE" = true ]; then + # library-nonet fails to build with coverage (Issue 13742) dub test --compiler=${DC} -b unittest-cov ./build.d -cov diff --git a/source/dub/internal/utils.d b/source/dub/internal/utils.d index b4e658d..538f034 100644 --- a/source/dub/internal/utils.d +++ b/source/dub/internal/utils.d @@ -23,9 +23,13 @@ import std.format; import std.string : format; import std.process; -import std.net.curl; -public import std.net.curl : HTTPStatusException; import std.traits : isIntegral; +version(DubUseCurl) +{ + import std.net.curl; + public import std.net.curl : HTTPStatusException; +} + private NativePath[] temporary_files; @@ -206,6 +210,9 @@ } } +version (Have_vibe_d_http) + public import vibe.http.common : HTTPStatusException; + /** Downloads a file from the specified URL. @@ -216,36 +223,48 @@ `timeout` ms, or if the average transfer rate drops below 10 bytes / s for more than `timeout` seconds. Pass `0` as `timeout` to disable both timeout mechanisms. + + Note: Timeouts are only implemented when curl is used (DubUseCurl). */ void download(string url, string filename, uint timeout = 8) { - auto conn = HTTP(); - setupHTTPClient(conn, timeout); - logDebug("Storing %s...", url); - std.net.curl.download(url, filename, conn); - // workaround https://issues.dlang.org/show_bug.cgi?id=18318 - auto sl = conn.statusLine; - logDebug("Download %s %s", url, sl); - if (sl.code / 100 != 2) - throw new HTTPStatusException(sl.code, - "Downloading %s failed with %d (%s).".format(url, sl.code, sl.reason)); + version(DubUseCurl) { + auto conn = HTTP(); + setupHTTPClient(conn, timeout); + logDebug("Storing %s...", url); + std.net.curl.download(url, filename, conn); + // workaround https://issues.dlang.org/show_bug.cgi?id=18318 + auto sl = conn.statusLine; + logDebug("Download %s %s", url, sl); + if (sl.code / 100 != 2) + throw new HTTPStatusException(sl.code, + "Downloading %s failed with %d (%s).".format(url, sl.code, sl.reason)); + } else version (Have_vibe_d_http) { + import vibe.inet.urltransfer; + vibe.inet.urltransfer.download(url, filename); + } else assert(false); } - /// ditto void download(URL url, NativePath filename, uint timeout = 8) { download(url.toString(), filename.toNativeString(), timeout); } - /// ditto ubyte[] download(string url, uint timeout = 8) { - auto conn = HTTP(); - setupHTTPClient(conn, timeout); - logDebug("Getting %s...", url); - return cast(ubyte[])get(url, conn); + version(DubUseCurl) { + auto conn = HTTP(); + setupHTTPClient(conn, timeout); + logDebug("Getting %s...", url); + return cast(ubyte[])get(url, conn); + } else version (Have_vibe_d_http) { + import vibe.inet.urltransfer; + import vibe.stream.operations; + ubyte[] ret; + vibe.inet.urltransfer.download(url, (scope input) { ret = input.readAll(); }); + return ret; + } else assert(false); } - /// ditto ubyte[] download(URL url, uint timeout = 8) { @@ -265,25 +284,44 @@ `timeout` ms, or if the average transfer rate drops below 10 bytes / s for more than `timeout` seconds. Pass `0` as `timeout` to disable both timeout mechanisms. + + Note: Timeouts are only implemented when curl is used (DubUseCurl). **/ void retryDownload(URL url, NativePath filename, size_t retryCount = 3, uint timeout = 8) { foreach(i; 0..retryCount) { - try { - download(url, filename, timeout); - return; - } - catch(HTTPStatusException e) { - if (e.status == 404) throw e; - else { + version(DubUseCurl) { + try { + download(url, filename, timeout); + return; + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } + catch(CurlException e) { logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); - if (i == retryCount - 1) throw e; - else continue; + continue; } } - catch(CurlException e) { - logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); - continue; + else + { + try { + download(url, filename); + return; + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } } } throw new Exception("Failed to download %s".format(url)); @@ -293,20 +331,36 @@ ubyte[] retryDownload(URL url, size_t retryCount = 3, uint timeout = 8) { foreach(i; 0..retryCount) { - try { - return download(url, timeout); - } - catch(HTTPStatusException e) { - if (e.status == 404) throw e; - else { + version(DubUseCurl) { + try { + return download(url, timeout); + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } + catch(CurlException e) { logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); - if (i == retryCount - 1) throw e; - else continue; + continue; } } - catch(CurlException e) { - logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); - continue; + else + { + try { + return download(url); + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } } } throw new Exception("Failed to download %s".format(url)); @@ -388,30 +442,32 @@ } -void setupHTTPClient(ref HTTP conn, uint timeout) -{ - static if( is(typeof(&conn.verifyPeer)) ) - conn.verifyPeer = false; +version(DubUseCurl) { + void setupHTTPClient(ref HTTP conn, uint timeout) + { + static if( is(typeof(&conn.verifyPeer)) ) + conn.verifyPeer = false; - auto proxy = environment.get("http_proxy", null); - if (proxy.length) conn.proxy = proxy; + auto proxy = environment.get("http_proxy", null); + if (proxy.length) conn.proxy = proxy; - auto noProxy = environment.get("no_proxy", null); - if (noProxy.length) conn.handle.set(CurlOption.noproxy, noProxy); + auto noProxy = environment.get("no_proxy", null); + if (noProxy.length) conn.handle.set(CurlOption.noproxy, noProxy); - conn.handle.set(CurlOption.encoding, ""); - if (timeout) { - // connection (TLS+TCP) times out after 8s - conn.handle.set(CurlOption.connecttimeout, timeout); - // transfers time out after 8s below 10 byte/s - conn.handle.set(CurlOption.low_speed_limit, 10); - conn.handle.set(CurlOption.low_speed_time, timeout); + conn.handle.set(CurlOption.encoding, ""); + if (timeout) { + // connection (TLS+TCP) times out after 8s + conn.handle.set(CurlOption.connecttimeout, timeout); + // transfers time out after 8s below 10 byte/s + conn.handle.set(CurlOption.low_speed_limit, 10); + conn.handle.set(CurlOption.low_speed_time, timeout); + } + + conn.addRequestHeader("User-Agent", "dub/"~getDUBVersion()~" (std.net.curl; +https://github.com/rejectedsoftware/dub)"); + + enum CURL_NETRC_OPTIONAL = 1; + conn.handle.set(CurlOption.netrc, CURL_NETRC_OPTIONAL); } - - conn.addRequestHeader("User-Agent", "dub/"~getDUBVersion()~" (std.net.curl; +https://github.com/rejectedsoftware/dub)"); - - enum CURL_NETRC_OPTIONAL = 1; - conn.handle.set(CurlOption.netrc, CURL_NETRC_OPTIONAL); } string stripUTF8Bom(string str)