diff --git a/source/dub/internal/utils.d b/source/dub/internal/utils.d index 3e36af7..ac62e30 100644 --- a/source/dub/internal/utils.d +++ b/source/dub/internal/utils.d @@ -325,6 +325,94 @@ return download(url.toString(), timeout); } +/** + Downloads a file from the specified URL with retry logic. + + Downloads a file from the specified URL with up to n tries on failure + Throws: `Exception` if the download failed or `HTTPStatusException` after the nth retry or + on "unrecoverable failures" such as 404 not found + Otherwise might throw anything else that `download` throws. + See_Also: download +**/ +void retryDownload(URL url, NativePath filename, size_t retryCount = 3) +{ + foreach(i; 0..retryCount) { + version(DubUseCurl) { + try { + download(url, filename); + return; + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } + catch(CurlException e) { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + continue; + } + } + else + { + try { + download(url, filename); + return; + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } + } + } + throw new Exception("Failed to download %s".format(url)); +} + +///ditto +ubyte[] retryDownload(URL url, size_t retryCount = 3) +{ + foreach(i; 0..retryCount) { + version(DubUseCurl) { + try { + return download(url); + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } + catch(CurlException e) { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + continue; + } + } + else + { + try { + return download(url); + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else { + logDebug("Failed to download %s (Attempt %s of %s)", url, i + 1, retryCount); + if (i == retryCount - 1) throw e; + else continue; + } + } + } + } + throw new Exception("Failed to download %s".format(url)); +} + /// Returns the current DUB version in semantic version format string getDUBVersion() { diff --git a/source/dub/packagesuppliers/maven.d b/source/dub/packagesuppliers/maven.d index 235aa32..372851d 100644 --- a/source/dub/packagesuppliers/maven.d +++ b/source/dub/packagesuppliers/maven.d @@ -9,7 +9,7 @@ to search for available packages. */ class MavenRegistryPackageSupplier : PackageSupplier { - import dub.internal.utils : download, HTTPStatusException; + import dub.internal.utils : retryDownload, HTTPStatusException; import dub.internal.vibecompat.data.json : serializeToJson; import dub.internal.vibecompat.core.log; import dub.internal.vibecompat.inet.url : URL; @@ -55,19 +55,17 @@ return; auto vers = best["version"].get!string; auto url = m_mavenUrl~NativePath("%s/%s/%s-%s.zip".format(packageId, vers, packageId, vers)); - logDiagnostic("Downloading from '%s'", url); - foreach(i; 0..3) { - try{ - download(url, path); - return; - } - catch(HTTPStatusException e) { - if (e.status == 404) throw e; - else { - logDebug("Failed to download package %s from %s (Attempt %s of 3)", packageId, url, i + 1); - continue; - } - } + + try { + retryDownload(url, path); + return; + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else logDebug("Failed to download package %s from %s", packageId, url); + } + catch(Exception e) { + logDebug("Failed to download package %s from %s", packageId, url); } throw new Exception("Failed to download package %s from %s".format(packageId, url)); } @@ -92,27 +90,16 @@ auto url = m_mavenUrl~NativePath(packageId~"/maven-metadata.xml"); logDebug("Downloading maven metadata for %s", packageId); - logDebug("Getting from %s", url); - string xmlData; - foreach(i; 0..3) { - try { - xmlData = cast(string)download(url); - break; + + try + xmlData = cast(string)retryDownload(url); + catch(HTTPStatusException e) { + if (e.status == 404) { + logDebug("Maven metadata %s not found at %s (404): %s", packageId, description, e.msg); + return Json(null); } - catch (HTTPStatusException e) - { - if (e.status == 404) { - logDebug("Maven metadata %s not found at %s (404): %s", packageId, description, e.msg); - return Json(null); - } - else { - logDebug("Error getting maven metadata for %s at %s (attempt %s of 3): %s", packageId, description, i + 1, e.msg); - if (i == 2) - throw e; - continue; - } - } + else throw e; } auto json = Json(["name": Json(packageId), "versions": Json.emptyArray]); diff --git a/source/dub/packagesuppliers/registry.d b/source/dub/packagesuppliers/registry.d index 6706b47..a91374d 100644 --- a/source/dub/packagesuppliers/registry.d +++ b/source/dub/packagesuppliers/registry.d @@ -11,7 +11,7 @@ $(LINK https://code.dlang.org/)) to search for available packages. */ class RegistryPackageSupplier : PackageSupplier { - import dub.internal.utils : download, HTTPStatusException; + import dub.internal.utils : download, retryDownload, HTTPStatusException; import dub.internal.vibecompat.core.log; import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson; import dub.internal.vibecompat.inet.url : URL; @@ -57,19 +57,16 @@ return; auto vers = best["version"].get!string; auto url = m_registryUrl ~ NativePath(PackagesPath~"/"~packageId~"/"~vers~".zip"); - logDiagnostic("Downloading from '%s'", url); - foreach(i; 0..3) { - try{ - download(url, path); - return; - } - catch(HTTPStatusException e) { - if (e.status == 404) throw e; - else { - logDebug("Failed to download package %s from %s (Attempt %s of 3)", packageId, url, i + 1); - continue; - } - } + try { + retryDownload(url, path); + return; + } + catch(HTTPStatusException e) { + if (e.status == 404) throw e; + else logDebug("Failed to download package %s from %s", packageId, url); + } + catch(Exception e) { + logDebug("Failed to download package %s from %s", packageId, url); } throw new Exception("Failed to download package %s from %s".format(packageId, url)); } @@ -92,28 +89,18 @@ auto url = m_registryUrl ~ NativePath(PackagesPath ~ "/" ~ packageId ~ ".json"); logDebug("Downloading metadata for %s", packageId); - logDebug("Getting from %s", url); - string jsonData; - foreach(i; 0..3) { - try { - jsonData = cast(string)download(url); - break; + + try + jsonData = cast(string)retryDownload(url); + catch(HTTPStatusException e) { + if (e.status == 404) { + logDebug("Package %s not found at %s (404): %s", packageId, description, e.msg); + return Json(null); } - catch (HTTPStatusException e) - { - if (e.status == 404) { - logDebug("Package %s not found at %s (404): %s", packageId, description, e.msg); - return Json(null); - } - else { - logDebug("Error getting metadata for package %s at %s (attempt %s of 3): %s", packageId, description, i + 1, e.msg); - if (i == 2) - throw e; - continue; - } - } + else throw e; } + Json json = parseJsonString(jsonData, url.toString()); // strip readme data (to save size and time) foreach (ref v; json["versions"])