Newer
Older
dub_jkp / source / dub / packagesupplier.d
  1. /**
  2. A package supplier, able to get some packages to the local FS.
  3.  
  4. Copyright: © 2012-2013 Matthias Dondorff
  5. License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
  6. Authors: Matthias Dondorff
  7. */
  8. module dub.packagesupplier;
  9.  
  10. import dub.dependency;
  11. import dub.internal.utils;
  12. import dub.internal.vibecompat.core.log;
  13. import dub.internal.vibecompat.core.file;
  14. import dub.internal.vibecompat.data.json;
  15. import dub.internal.vibecompat.inet.url;
  16.  
  17. import std.algorithm : filter, sort;
  18. import std.array : array;
  19. import std.conv;
  20. import std.datetime;
  21. import std.exception;
  22. import std.file;
  23. import std.string : format;
  24. import std.zip;
  25.  
  26. // TODO: drop the "best package" behavior and let retrievePackage/getPackageDescription take a Version instead of Dependency
  27.  
  28. /// Supplies packages, this is done by supplying the latest possible version
  29. /// which is available.
  30. interface PackageSupplier {
  31. /// Returns a hunman readable representation of the supplier
  32. @property string description();
  33.  
  34. Version[] getVersions(string package_id);
  35.  
  36. /// path: absolute path to store the package (usually in a zip format)
  37. void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release);
  38.  
  39. /// returns the metadata for the package
  40. Json getPackageDescription(string packageId, Dependency dep, bool pre_release);
  41.  
  42. static struct SearchResult { string name, description, version_; }
  43. SearchResult[] searchPackages(string query);
  44. }
  45.  
  46. class FileSystemPackageSupplier : PackageSupplier {
  47. private {
  48. Path m_path;
  49. }
  50.  
  51. this(Path root) { m_path = root; }
  52.  
  53. override @property string description() { return "file repository at "~m_path.toNativeString(); }
  54.  
  55. Version[] getVersions(string package_id)
  56. {
  57. Version[] ret;
  58. foreach (DirEntry d; dirEntries(m_path.toNativeString(), package_id~"*", SpanMode.shallow)) {
  59. Path p = Path(d.name);
  60. logDebug("Entry: %s", p);
  61. enforce(to!string(p.head)[$-4..$] == ".zip");
  62. auto vers = p.head.toString()[package_id.length+1..$-4];
  63. logDebug("Version: %s", vers);
  64. ret ~= Version(vers);
  65. }
  66. ret.sort();
  67. return ret;
  68. }
  69.  
  70. void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release)
  71. {
  72. enforce(path.absolute);
  73. logInfo("Storing package '"~packageId~"', version requirements: %s", dep);
  74. auto filename = bestPackageFile(packageId, dep, pre_release);
  75. enforce(existsFile(filename));
  76. copyFile(filename, path);
  77. }
  78.  
  79. Json getPackageDescription(string packageId, Dependency dep, bool pre_release)
  80. {
  81. auto filename = bestPackageFile(packageId, dep, pre_release);
  82. return jsonFromZip(filename, "dub.json");
  83. }
  84.  
  85. SearchResult[] searchPackages(string query) {
  86. return null;
  87. }
  88.  
  89. private Path bestPackageFile(string packageId, Dependency dep, bool pre_release)
  90. {
  91. Path toPath(Version ver) {
  92. return m_path ~ (packageId ~ "-" ~ ver.toString() ~ ".zip");
  93. }
  94. auto versions = getVersions(packageId).filter!(v => dep.matches(v)).array;
  95. enforce(versions.length > 0, format("No package %s found matching %s", packageId, dep));
  96. foreach_reverse (ver; versions) {
  97. if (pre_release || !ver.isPreRelease)
  98. return toPath(ver);
  99. }
  100. return toPath(versions[$-1]);
  101. }
  102. }
  103.  
  104.  
  105. /// Client PackageSupplier using the registry available via registerVpmRegistry
  106. class RegistryPackageSupplier : PackageSupplier {
  107. private {
  108. URL m_registryUrl;
  109. struct CacheEntry { Json data; SysTime cacheTime; }
  110. CacheEntry[string] m_metadataCache;
  111. Duration m_maxCacheTime;
  112. }
  113.  
  114. this(URL registry)
  115. {
  116. m_registryUrl = registry;
  117. m_maxCacheTime = 24.hours();
  118. }
  119.  
  120. override @property string description() { return "registry at "~m_registryUrl.toString(); }
  121.  
  122. Version[] getVersions(string package_id)
  123. {
  124. Version[] ret;
  125. Json md = getMetadata(package_id);
  126. foreach (json; md["versions"]) {
  127. auto cur = Version(cast(string)json["version"]);
  128. ret ~= cur;
  129. }
  130. ret.sort();
  131. return ret;
  132. }
  133.  
  134. void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release)
  135. {
  136. import std.array : replace;
  137. Json best = getBestPackage(packageId, dep, pre_release);
  138. auto vers = best["version"].get!string;
  139. auto url = m_registryUrl ~ Path(PackagesPath~"/"~packageId~"/"~vers~".zip");
  140. logDiagnostic("Found download URL: '%s'", url);
  141. download(url, path);
  142. }
  143.  
  144. Json getPackageDescription(string packageId, Dependency dep, bool pre_release)
  145. {
  146. return getBestPackage(packageId, dep, pre_release);
  147. }
  148.  
  149. private Json getMetadata(string packageId)
  150. {
  151. auto now = Clock.currTime(UTC());
  152. if (auto pentry = packageId in m_metadataCache) {
  153. if (pentry.cacheTime + m_maxCacheTime > now)
  154. return pentry.data;
  155. m_metadataCache.remove(packageId);
  156. }
  157.  
  158. auto url = m_registryUrl ~ Path(PackagesPath ~ "/" ~ packageId ~ ".json");
  159.  
  160. logDebug("Downloading metadata for %s", packageId);
  161. logDebug("Getting from %s", url);
  162.  
  163. auto jsonData = cast(string)download(url);
  164. Json json = parseJsonString(jsonData, url.toString());
  165. // strip readme data (to save size and time)
  166. foreach (ref v; json["versions"])
  167. v.remove("readme");
  168. m_metadataCache[packageId] = CacheEntry(json, now);
  169. return json;
  170. }
  171.  
  172. SearchResult[] searchPackages(string query) {
  173. import std.uri : encodeComponent;
  174. auto url = m_registryUrl;
  175. url.localURI = "/api/packages/search?q="~encodeComponent(query);
  176. string data;
  177. try
  178. data = cast(string)download(url);
  179. catch (Exception)
  180. return null;
  181. import std.algorithm : map;
  182. return data.parseJson.opt!(Json[])
  183. .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string))
  184. .array;
  185. }
  186.  
  187. private Json getBestPackage(string packageId, Dependency dep, bool pre_release)
  188. {
  189. Json md = getMetadata(packageId);
  190. Json best = null;
  191. Version bestver;
  192. foreach (json; md["versions"]) {
  193. auto cur = Version(cast(string)json["version"]);
  194. if (!dep.matches(cur)) continue;
  195. if (best == null) best = json;
  196. else if (pre_release) {
  197. if (cur > bestver) best = json;
  198. } else if (bestver.isPreRelease) {
  199. if (!cur.isPreRelease || cur > bestver) best = json;
  200. } else if (!cur.isPreRelease && cur > bestver) best = json;
  201. bestver = Version(cast(string)best["version"]);
  202. }
  203. enforce(best != null, "No package candidate found for "~packageId~" "~dep.toString());
  204. return best;
  205. }
  206. }
  207.  
  208. private enum PackagesPath = "packages";