Newer
Older
dub_jkp / source / dub / packagesuppliers / registry.d
  1. module dub.packagesuppliers.registry;
  2.  
  3. import dub.dependency;
  4. import dub.packagesuppliers.packagesupplier;
  5.  
  6. package enum PackagesPath = "packages";
  7.  
  8. /**
  9. Online registry based package supplier.
  10.  
  11. This package supplier connects to an online registry (e.g.
  12. $(LINK https://code.dlang.org/)) to search for available packages.
  13. */
  14. class RegistryPackageSupplier : PackageSupplier {
  15. import dub.internal.utils : retryDownload, HTTPStatusException;
  16. import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson;
  17. import dub.internal.vibecompat.inet.url : URL;
  18. import dub.internal.logging;
  19.  
  20. import std.uri : encodeComponent;
  21. import std.datetime : Clock, Duration, hours, SysTime, UTC;
  22. private {
  23. URL m_registryUrl;
  24. struct CacheEntry { Json data; SysTime cacheTime; }
  25. CacheEntry[PackageName] m_metadataCache;
  26. Duration m_maxCacheTime;
  27. }
  28.  
  29. this(URL registry)
  30. {
  31. m_registryUrl = registry;
  32. m_maxCacheTime = 24.hours();
  33. }
  34.  
  35. override @property string description() { return "registry at "~m_registryUrl.toString(); }
  36.  
  37. override Version[] getVersions(in PackageName name)
  38. {
  39. import std.algorithm.sorting : sort;
  40. auto md = getMetadata(name);
  41. if (md.type == Json.Type.null_)
  42. return null;
  43. Version[] ret;
  44. foreach (json; md["versions"]) {
  45. auto cur = Version(cast(string)json["version"]);
  46. ret ~= cur;
  47. }
  48. ret.sort();
  49. return ret;
  50. }
  51.  
  52. auto genPackageDownloadUrl(in PackageName name, in VersionRange dep, bool pre_release)
  53. {
  54. import std.array : replace;
  55. import std.format : format;
  56. import std.typecons : Nullable;
  57. auto md = getMetadata(name);
  58. Json best = getBestPackage(md, name, dep, pre_release);
  59. Nullable!URL ret;
  60. if (best.type != Json.Type.null_)
  61. {
  62. auto vers = best["version"].get!string;
  63. ret = m_registryUrl ~ NativePath(
  64. "%s/%s/%s.zip".format(PackagesPath, name.main, vers));
  65. }
  66. return ret;
  67. }
  68.  
  69. override ubyte[] fetchPackage(in PackageName name,
  70. in VersionRange dep, bool pre_release)
  71. {
  72. import std.format : format;
  73.  
  74. auto url = genPackageDownloadUrl(name, dep, pre_release);
  75. if(url.isNull) return null;
  76. try {
  77. return retryDownload(url.get);
  78. }
  79. catch(HTTPStatusException e) {
  80. if (e.status == 404) throw e;
  81. else logDebug("Failed to download package %s from %s", name.main, url);
  82. }
  83. catch(Exception e) {
  84. logDebug("Failed to download package %s from %s", name.main, url);
  85. }
  86. throw new Exception("Failed to download package %s from %s".format(name.main, url));
  87. }
  88.  
  89. override Json fetchPackageRecipe(in PackageName name, in VersionRange dep,
  90. bool pre_release)
  91. {
  92. auto md = getMetadata(name);
  93. return getBestPackage(md, name, dep, pre_release);
  94. }
  95.  
  96. private Json getMetadata(in PackageName name)
  97. {
  98. auto now = Clock.currTime(UTC());
  99. if (auto pentry = name.main in m_metadataCache) {
  100. if (pentry.cacheTime + m_maxCacheTime > now)
  101. return pentry.data;
  102. m_metadataCache.remove(name.main);
  103. }
  104.  
  105. auto url = m_registryUrl ~ NativePath("api/packages/infos");
  106.  
  107. url.queryString = "packages=" ~
  108. encodeComponent(`["` ~ name.main.toString() ~ `"]`) ~
  109. "&include_dependencies=true&minimize=true";
  110.  
  111. logDebug("Downloading metadata for %s", name.main);
  112. string jsonData;
  113.  
  114. jsonData = cast(string)retryDownload(url);
  115.  
  116. Json json = parseJsonString(jsonData, url.toString());
  117. foreach (pkg, info; json.get!(Json[string]))
  118. {
  119. logDebug("adding %s to metadata cache", pkg);
  120. m_metadataCache[PackageName(pkg)] = CacheEntry(info, now);
  121. }
  122. return json[name.main.toString()];
  123. }
  124.  
  125. SearchResult[] searchPackages(string query) {
  126. import std.array : array;
  127. import std.algorithm.iteration : map;
  128. import std.uri : encodeComponent;
  129. auto url = m_registryUrl;
  130. url.localURI = "/api/packages/search?q="~encodeComponent(query);
  131. string data;
  132. data = cast(string)retryDownload(url);
  133. return data.parseJson.opt!(Json[])
  134. .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string))
  135. .array;
  136. }
  137. }