Newer
Older
dub_jkp / source / dub / packagemanager.d
  1. /**
  2. Management of packages on the local computer.
  3.  
  4. Copyright: © 2012-2016 rejectedsoftware e.K.
  5. License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
  6. Authors: Sönke Ludwig, Matthias Dondorff
  7. */
  8. module dub.packagemanager;
  9.  
  10. import dub.dependency;
  11. import dub.internal.utils;
  12. import dub.internal.vibecompat.core.file : FileInfo;
  13. import dub.internal.vibecompat.data.json;
  14. import dub.internal.vibecompat.inet.path;
  15. import dub.internal.logging;
  16. import dub.package_;
  17. import dub.recipe.io;
  18. import dub.internal.configy.Exceptions;
  19. public import dub.internal.configy.Read : StrictMode;
  20.  
  21. import dub.internal.dyaml.stdsumtype;
  22.  
  23. import std.algorithm : countUntil, filter, map, sort, canFind, remove;
  24. import std.array;
  25. import std.conv;
  26. import std.digest.sha;
  27. import std.encoding : sanitize;
  28. import std.exception;
  29. import std.file;
  30. import std.range;
  31. import std.string;
  32. import std.zip;
  33.  
  34.  
  35. /// Indicates where a package has been or should be placed to.
  36. public enum PlacementLocation {
  37. /// Packages retrieved with 'local' will be placed in the current folder
  38. /// using the package name as destination.
  39. local,
  40. /// Packages with 'userWide' will be placed in a folder accessible by
  41. /// all of the applications from the current user.
  42. user,
  43. /// Packages retrieved with 'systemWide' will be placed in a shared folder,
  44. /// which can be accessed by all users of the system.
  45. system,
  46. }
  47.  
  48. /// Converts a `PlacementLocation` to a string
  49. public string toString (PlacementLocation loc) @safe pure nothrow @nogc
  50. {
  51. final switch (loc) {
  52. case PlacementLocation.local:
  53. return "Local";
  54. case PlacementLocation.user:
  55. return "User";
  56. case PlacementLocation.system:
  57. return "System";
  58. }
  59. }
  60.  
  61. /// The PackageManager can retrieve present packages and get / remove
  62. /// packages.
  63. class PackageManager {
  64. protected {
  65. /**
  66. * The 'internal' location, for packages not attributable to a location.
  67. *
  68. * There are two uses for this:
  69. * - In `bare` mode, the search paths are set at this scope,
  70. * and packages gathered are stored in `localPackage`;
  71. * - In the general case, any path-based or SCM-based dependency
  72. * is loaded in `fromPath`;
  73. */
  74. Location m_internal;
  75. /**
  76. * List of locations that are managed by this `PackageManager`
  77. *
  78. * The `PackageManager` can be instantiated either in 'bare' mode,
  79. * in which case this array will be empty, or in the normal mode,
  80. * this array will have 3 entries, matching values
  81. * in the `PlacementLocation` enum.
  82. *
  83. * See_Also: `Location`, `PlacementLocation`
  84. */
  85. Location[] m_repositories;
  86. /**
  87. * Whether `refresh` has been called or not
  88. *
  89. * Dub versions because v1.31 eagerly scan all available repositories,
  90. * leading to slowdown for the most common operation - `dub build` with
  91. * already resolved dependencies.
  92. * From v1.31 onward, those locations are not scanned eagerly,
  93. * unless one of the function requiring eager scanning does,
  94. * such as `getBestPackage` - as it needs to iterate the list
  95. * of available packages.
  96. */
  97. bool m_initialized;
  98. }
  99.  
  100. /**
  101. Instantiate an instance with a single search path
  102.  
  103. This constructor is used when dub is invoked with the '--bare' CLI switch.
  104. The instance will not look up the default repositories
  105. (e.g. ~/.dub/packages), using only `path` instead.
  106.  
  107. Params:
  108. path = Path of the single repository
  109. */
  110. this(NativePath path)
  111. {
  112. this.m_internal.searchPath = [ path ];
  113. this.refresh();
  114. }
  115.  
  116. this(NativePath package_path, NativePath user_path, NativePath system_path, bool refresh_packages = true)
  117. {
  118. m_repositories = [
  119. Location(package_path ~ ".dub/packages/"),
  120. Location(user_path ~ "packages/"),
  121. Location(system_path ~ "packages/")];
  122.  
  123. if (refresh_packages) refresh();
  124. }
  125.  
  126. /** Gets/sets the list of paths to search for local packages.
  127. */
  128. @property void searchPath(NativePath[] paths)
  129. {
  130. if (paths == this.m_internal.searchPath) return;
  131. this.m_internal.searchPath = paths.dup;
  132. this.refresh();
  133. }
  134. /// ditto
  135. @property const(NativePath)[] searchPath() const { return this.m_internal.searchPath; }
  136.  
  137. /** Returns the effective list of search paths, including default ones.
  138. */
  139. deprecated("Use the `PackageManager` facilities instead")
  140. @property const(NativePath)[] completeSearchPath()
  141. const {
  142. auto ret = appender!(const(NativePath)[])();
  143. ret.put(this.m_internal.searchPath);
  144. foreach (ref repo; m_repositories) {
  145. ret.put(repo.searchPath);
  146. ret.put(repo.packagePath);
  147. }
  148. return ret.data;
  149. }
  150.  
  151. /** Sets additional (read-only) package cache paths to search for packages.
  152.  
  153. Cache paths have the same structure as the default cache paths, such as
  154. ".dub/packages/".
  155.  
  156. Note that previously set custom paths will be removed when setting this
  157. property.
  158. */
  159. @property void customCachePaths(NativePath[] custom_cache_paths)
  160. {
  161. import std.algorithm.iteration : map;
  162. import std.array : array;
  163.  
  164. m_repositories.length = PlacementLocation.max+1;
  165. m_repositories ~= custom_cache_paths.map!(p => Location(p)).array;
  166.  
  167. this.refresh();
  168. }
  169.  
  170. /**
  171. * Looks up a package, first in the list of loaded packages,
  172. * then directly on the file system.
  173. *
  174. * This function allows for lazy loading of packages, without needing to
  175. * first scan all the available locations (as `refresh` does).
  176. *
  177. * Note:
  178. * This function does not take overrides into account. Overrides need
  179. * to be resolved by the caller before `lookup` is called.
  180. * Additionally, if a package of the same version is loaded in multiple
  181. * locations, the first one matching (local > user > system)
  182. * will be returned.
  183. *
  184. * Params:
  185. * name = The full name of the package to look up
  186. * vers = The version the package must match
  187. *
  188. * Returns:
  189. * A `Package` if one was found, `null` if none exists.
  190. */
  191. protected Package lookup (in PackageName name, in Version vers) {
  192. if (!this.m_initialized)
  193. this.refresh();
  194.  
  195. if (auto pkg = this.m_internal.lookup(name, vers))
  196. return pkg;
  197.  
  198. foreach (ref location; this.m_repositories)
  199. if (auto p = location.load(name, vers, this))
  200. return p;
  201.  
  202. return null;
  203. }
  204.  
  205. /** Looks up a specific package.
  206.  
  207. Looks up a package matching the given version/path in the set of
  208. registered packages. The lookup order is done according the the
  209. usual rules (see getPackageIterator).
  210.  
  211. Params:
  212. name = The name of the package
  213. ver = The exact version of the package to query
  214. path = An exact path that the package must reside in. Note that
  215. the package must still be registered in the package manager.
  216. enable_overrides = Apply the local package override list before
  217. returning a package (enabled by default)
  218.  
  219. Returns:
  220. The matching package or null if no match was found.
  221. */
  222. Package getPackage(in PackageName name, in Version ver, bool enable_overrides = true)
  223. {
  224. if (enable_overrides) {
  225. foreach (ref repo; m_repositories)
  226. foreach (ovr; repo.overrides)
  227. if (ovr.package_ == name.toString() && ovr.source.matches(ver)) {
  228. Package pack = ovr.target.match!(
  229. (NativePath path) => getOrLoadPackage(path),
  230. (Version vers) => getPackage(name, vers, false),
  231. );
  232. if (pack) return pack;
  233.  
  234. ovr.target.match!(
  235. (any) {
  236. logWarn("Package override %s %s -> '%s' doesn't reference an existing package.",
  237. ovr.package_, ovr.source, any);
  238. },
  239. );
  240. }
  241. }
  242.  
  243. return this.lookup(name, ver);
  244. }
  245.  
  246. deprecated("Use the overload that accepts a `PackageName` instead")
  247. Package getPackage(string name, Version ver, bool enable_overrides = true)
  248. {
  249. return this.getPackage(PackageName(name), ver, enable_overrides);
  250. }
  251.  
  252. /// ditto
  253. deprecated("Use the overload that accepts a `Version` as second argument")
  254. Package getPackage(string name, string ver, bool enable_overrides = true)
  255. {
  256. return getPackage(name, Version(ver), enable_overrides);
  257. }
  258.  
  259. /// ditto
  260. deprecated("Use the overload that takes a `PlacementLocation`")
  261. Package getPackage(string name, Version ver, NativePath path)
  262. {
  263. foreach (p; getPackageIterator(name)) {
  264. auto pvm = isManagedPackage(p) ? VersionMatchMode.strict : VersionMatchMode.standard;
  265. if (p.version_.matches(ver, pvm) && p.path.startsWith(path))
  266. return p;
  267. }
  268. return null;
  269. }
  270.  
  271. /// Ditto
  272. deprecated("Use the overload that accepts a `PackageName` instead")
  273. Package getPackage(string name, Version ver, PlacementLocation loc)
  274. {
  275. return this.getPackage(PackageName(name), ver, loc);
  276. }
  277.  
  278. /// Ditto
  279. Package getPackage(in PackageName name, in Version ver, PlacementLocation loc)
  280. {
  281. // Bare mode
  282. if (loc >= this.m_repositories.length)
  283. return null;
  284. return this.m_repositories[loc].load(name, ver, this);
  285. }
  286.  
  287. /// ditto
  288. deprecated("Use the overload that accepts a `Version` as second argument")
  289. Package getPackage(string name, string ver, NativePath path)
  290. {
  291. return getPackage(name, Version(ver), path);
  292. }
  293.  
  294. /// ditto
  295. deprecated("Use another `PackageManager` API, open an issue if none suits you")
  296. Package getPackage(string name, NativePath path)
  297. {
  298. foreach( p; getPackageIterator(name) )
  299. if (p.path.startsWith(path))
  300. return p;
  301. return null;
  302. }
  303.  
  304.  
  305. /** Looks up the first package matching the given name.
  306. */
  307. deprecated("Use `getBestPackage` instead")
  308. Package getFirstPackage(string name)
  309. {
  310. foreach (ep; getPackageIterator(name))
  311. return ep;
  312. return null;
  313. }
  314.  
  315. /** Looks up the latest package matching the given name.
  316. */
  317. deprecated("Use `getBestPackage` with `name, Dependency.any` instead")
  318. Package getLatestPackage(string name)
  319. {
  320. Package pkg;
  321. foreach (ep; getPackageIterator(name))
  322. if (pkg is null || pkg.version_ < ep.version_)
  323. pkg = ep;
  324. return pkg;
  325. }
  326.  
  327. /** For a given package path, returns the corresponding package.
  328.  
  329. If the package is already loaded, a reference is returned. Otherwise
  330. the package gets loaded and cached for the next call to this function.
  331.  
  332. Params:
  333. path = NativePath to the root directory of the package
  334. recipe_path = Optional path to the recipe file of the package
  335. allow_sub_packages = Also return a sub package if it resides in the given folder
  336. mode = Whether to issue errors, warning, or ignore unknown keys in dub.json
  337.  
  338. Returns: The packages loaded from the given path
  339. Throws: Throws an exception if no package can be loaded
  340. */
  341. Package getOrLoadPackage(NativePath path, NativePath recipe_path = NativePath.init,
  342. bool allow_sub_packages = false, StrictMode mode = StrictMode.Ignore)
  343. {
  344. path.endsWithSlash = true;
  345. foreach (p; this.m_internal.fromPath)
  346. if (p.path == path && (!p.parentPackage || (allow_sub_packages && p.parentPackage.path != p.path)))
  347. return p;
  348. auto pack = this.load(path, recipe_path, null, null, mode);
  349. addPackages(this.m_internal.fromPath, pack);
  350. return pack;
  351. }
  352.  
  353. /**
  354. * Loads a `Package` from the filesystem
  355. *
  356. * This is called when a `Package` needs to be loaded from the path.
  357. * This does not change the internal state of the `PackageManager`,
  358. * it simply loads the `Package` and returns it - it is up to the caller
  359. * to call `addPackages`.
  360. *
  361. * Throws:
  362. * If no package can be found at the `path` / with the `recipe`.
  363. *
  364. * Params:
  365. * path = The directory in which the package resides.
  366. * recipe = Optional path to the package recipe file. If left empty,
  367. * the `path` directory will be searched for a recipe file.
  368. * parent = Reference to the parent package, if the new package is a
  369. * sub package.
  370. * version_ = Optional version to associate to the package instead of
  371. * the one declared in the package recipe, or the one
  372. * determined by invoking the VCS (GIT currently).
  373. * mode = Whether to issue errors, warning, or ignore unknown keys in
  374. * dub.json
  375. *
  376. * Returns: A populated `Package`.
  377. */
  378. protected Package load(NativePath path, NativePath recipe = NativePath.init,
  379. Package parent = null, string version_ = null,
  380. StrictMode mode = StrictMode.Ignore)
  381. {
  382. if (recipe.empty)
  383. recipe = Package.findPackageFile(path);
  384.  
  385. enforce(!recipe.empty,
  386. "No package file found in %s, expected one of %s"
  387. .format(path.toNativeString(),
  388. packageInfoFiles.map!(f => cast(string)f.filename).join("/")));
  389.  
  390. const PackageName pname = parent
  391. ? PackageName(parent.name) : PackageName.init;
  392. auto content = readPackageRecipe(recipe, pname, mode);
  393. auto ret = new Package(content, path, parent, version_);
  394. ret.m_infoFile = recipe;
  395. return ret;
  396. }
  397.  
  398. /** For a given SCM repository, returns the corresponding package.
  399.  
  400. An SCM repository is provided as its remote URL, the repository is cloned
  401. and in the dependency specified commit is checked out.
  402.  
  403. If the target directory already exists, just returns the package
  404. without cloning.
  405.  
  406. Params:
  407. name = Package name
  408. dependency = Dependency that contains the repository URL and a specific commit
  409.  
  410. Returns:
  411. The package loaded from the given SCM repository or null if the
  412. package couldn't be loaded.
  413. */
  414. Package loadSCMPackage(in PackageName name, in Repository repo)
  415. in { assert(!repo.empty); }
  416. do {
  417. Package pack;
  418.  
  419. final switch (repo.kind)
  420. {
  421. case repo.Kind.git:
  422. pack = loadGitPackage(name, repo);
  423. }
  424. if (pack !is null) {
  425. addPackages(this.m_internal.fromPath, pack);
  426. }
  427. return pack;
  428. }
  429.  
  430. deprecated("Use the overload that accepts a `dub.dependency : Repository`")
  431. Package loadSCMPackage(string name, Dependency dependency)
  432. in { assert(!dependency.repository.empty); }
  433. do { return this.loadSCMPackage(name, dependency.repository); }
  434.  
  435. deprecated("Use `loadSCMPackage(PackageName, Repository)`")
  436. Package loadSCMPackage(string name, Repository repo)
  437. {
  438. return this.loadSCMPackage(PackageName(name), repo);
  439. }
  440.  
  441. private Package loadGitPackage(in PackageName name, in Repository repo)
  442. {
  443. import dub.internal.git : cloneRepository;
  444.  
  445. if (!repo.ref_.startsWith("~") && !repo.ref_.isGitHash) {
  446. return null;
  447. }
  448.  
  449. string gitReference = repo.ref_.chompPrefix("~");
  450. NativePath destination = this.getPackagePath(PlacementLocation.user, name, repo.ref_);
  451.  
  452. foreach (p; getPackageIterator(name.toString())) {
  453. if (p.path == destination) {
  454. return p;
  455. }
  456. }
  457.  
  458. if (!cloneRepository(repo.remote, gitReference, destination.toNativeString())) {
  459. return null;
  460. }
  461.  
  462. return this.load(destination);
  463. }
  464.  
  465. /**
  466. * Get the final destination a specific package needs to be stored in.
  467. *
  468. * See `Location.getPackagePath`.
  469. */
  470. package(dub) NativePath getPackagePath(PlacementLocation base, in PackageName name, string vers)
  471. {
  472. assert(this.m_repositories.length == 3, "getPackagePath called in bare mode");
  473. return this.m_repositories[base].getPackagePath(name, vers);
  474. }
  475.  
  476. /**
  477. * Searches for the latest version of a package matching the version range.
  478. *
  479. * This will search the local file system only (it doesn't connect
  480. * to the registry) for the "best" (highest version) that matches `range`.
  481. * An overload with a single version exists to search for an exact version.
  482. *
  483. * Params:
  484. * name = Package name to search for
  485. * vers = Exact version to search for
  486. * range = Range of versions to search for, defaults to any
  487. *
  488. * Returns:
  489. * The best package matching the parameters, or `null` if none was found.
  490. */
  491. deprecated("Use the overload that accepts a `PackageName` instead")
  492. Package getBestPackage(string name, Version vers)
  493. {
  494. return this.getBestPackage(PackageName(name), vers);
  495. }
  496.  
  497. /// Ditto
  498. Package getBestPackage(in PackageName name, in Version vers)
  499. {
  500. return this.getBestPackage(name, VersionRange(vers, vers));
  501. }
  502.  
  503. /// Ditto
  504. deprecated("Use the overload that accepts a `PackageName` instead")
  505. Package getBestPackage(string name, VersionRange range = VersionRange.Any)
  506. {
  507. return this.getBestPackage(PackageName(name), range);
  508. }
  509.  
  510. /// Ditto
  511. Package getBestPackage(in PackageName name, in VersionRange range = VersionRange.Any)
  512. {
  513. return this.getBestPackage_(name, Dependency(range));
  514. }
  515.  
  516. /// Ditto
  517. deprecated("Use the overload that accepts a `Version` or a `VersionRange`")
  518. Package getBestPackage(string name, string range)
  519. {
  520. return this.getBestPackage(name, VersionRange.fromString(range));
  521. }
  522.  
  523. /// Ditto
  524. deprecated("`getBestPackage` should only be used with a `Version` or `VersionRange` argument")
  525. Package getBestPackage(string name, Dependency version_spec, bool enable_overrides = true)
  526. {
  527. return this.getBestPackage_(PackageName(name), version_spec, enable_overrides);
  528. }
  529.  
  530. // TODO: Merge this into `getBestPackage(string, VersionRange)`
  531. private Package getBestPackage_(in PackageName name, in Dependency version_spec,
  532. bool enable_overrides = true)
  533. {
  534. Package ret;
  535. foreach (p; getPackageIterator(name.toString())) {
  536. auto vmm = isManagedPackage(p) ? VersionMatchMode.strict : VersionMatchMode.standard;
  537. if (version_spec.matches(p.version_, vmm) && (!ret || p.version_ > ret.version_))
  538. ret = p;
  539. }
  540.  
  541. if (enable_overrides && ret) {
  542. if (auto ovr = getPackage(name, ret.version_))
  543. return ovr;
  544. }
  545. return ret;
  546. }
  547.  
  548. /** Gets the a specific sub package.
  549.  
  550. Params:
  551. base_package = The package from which to get a sub package
  552. sub_name = Name of the sub package (not prefixed with the base
  553. package name)
  554. silent_fail = If set to true, the function will return `null` if no
  555. package is found. Otherwise will throw an exception.
  556.  
  557. */
  558. Package getSubPackage(Package base_package, string sub_name, bool silent_fail)
  559. {
  560. foreach (p; getPackageIterator(base_package.name~":"~sub_name))
  561. if (p.parentPackage is base_package)
  562. return p;
  563. enforce(silent_fail, "Sub package \""~base_package.name~":"~sub_name~"\" doesn't exist.");
  564. return null;
  565. }
  566.  
  567.  
  568. /** Determines if a package is managed by DUB.
  569.  
  570. Managed packages can be upgraded and removed.
  571. */
  572. bool isManagedPackage(const(Package) pack)
  573. const {
  574. auto ppath = pack.basePackage.path;
  575. return isManagedPath(ppath);
  576. }
  577.  
  578. /** Determines if a specific path is within a DUB managed package folder.
  579.  
  580. By default, managed folders are "~/.dub/packages" and
  581. "/var/lib/dub/packages".
  582. */
  583. bool isManagedPath(NativePath path)
  584. const {
  585. foreach (rep; m_repositories)
  586. if (rep.isManaged(path))
  587. return true;
  588. return false;
  589. }
  590.  
  591. /** Enables iteration over all known local packages.
  592.  
  593. Returns: A delegate suitable for use with `foreach` is returned.
  594. */
  595. int delegate(int delegate(ref Package)) getPackageIterator()
  596. {
  597. // See `m_initialized` documentation
  598. if (!this.m_initialized)
  599. this.refresh();
  600.  
  601. int iterator(int delegate(ref Package) del)
  602. {
  603. // Search scope by priority, internal has the highest
  604. foreach (p; this.m_internal.fromPath)
  605. if (auto ret = del(p)) return ret;
  606. foreach (p; this.m_internal.localPackages)
  607. if (auto ret = del(p)) return ret;
  608.  
  609. foreach (ref repo; m_repositories) {
  610. foreach (p; repo.localPackages)
  611. if (auto ret = del(p)) return ret;
  612. foreach (p; repo.fromPath)
  613. if (auto ret = del(p)) return ret;
  614. }
  615. return 0;
  616. }
  617.  
  618. return &iterator;
  619. }
  620.  
  621. /** Enables iteration over all known local packages with a certain name.
  622.  
  623. Returns: A delegate suitable for use with `foreach` is returned.
  624. */
  625. int delegate(int delegate(ref Package)) getPackageIterator(string name)
  626. {
  627. int iterator(int delegate(ref Package) del)
  628. {
  629. foreach (p; getPackageIterator())
  630. if (p.name == name)
  631. if (auto ret = del(p)) return ret;
  632. return 0;
  633. }
  634.  
  635. return &iterator;
  636. }
  637.  
  638.  
  639. /** Returns a list of all package overrides for the given scope.
  640. */
  641. deprecated(OverrideDepMsg)
  642. const(PackageOverride)[] getOverrides(PlacementLocation scope_)
  643. const {
  644. return cast(typeof(return)) this.getOverrides_(scope_);
  645. }
  646.  
  647. package(dub) const(PackageOverride_)[] getOverrides_(PlacementLocation scope_)
  648. const {
  649. return m_repositories[scope_].overrides;
  650. }
  651.  
  652. /** Adds a new override for the given package.
  653. */
  654. deprecated("Use the overload that accepts a `VersionRange` as 3rd argument")
  655. void addOverride(PlacementLocation scope_, string package_, Dependency version_spec, Version target)
  656. {
  657. m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target);
  658. m_repositories[scope_].writeOverrides();
  659. }
  660. /// ditto
  661. deprecated("Use the overload that accepts a `VersionRange` as 3rd argument")
  662. void addOverride(PlacementLocation scope_, string package_, Dependency version_spec, NativePath target)
  663. {
  664. m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target);
  665. m_repositories[scope_].writeOverrides();
  666. }
  667.  
  668. /// Ditto
  669. deprecated(OverrideDepMsg)
  670. void addOverride(PlacementLocation scope_, string package_, VersionRange source, Version target)
  671. {
  672. this.addOverride_(scope_, package_, source, target);
  673. }
  674. /// ditto
  675. deprecated(OverrideDepMsg)
  676. void addOverride(PlacementLocation scope_, string package_, VersionRange source, NativePath target)
  677. {
  678. this.addOverride_(scope_, package_, source, target);
  679. }
  680.  
  681. // Non deprecated version that is used by `commandline`. Do not use!
  682. package(dub) void addOverride_(PlacementLocation scope_, string package_, VersionRange source, Version target)
  683. {
  684. m_repositories[scope_].overrides ~= PackageOverride_(package_, source, target);
  685. m_repositories[scope_].writeOverrides();
  686. }
  687. // Non deprecated version that is used by `commandline`. Do not use!
  688. package(dub) void addOverride_(PlacementLocation scope_, string package_, VersionRange source, NativePath target)
  689. {
  690. m_repositories[scope_].overrides ~= PackageOverride_(package_, source, target);
  691. m_repositories[scope_].writeOverrides();
  692. }
  693.  
  694. /** Removes an existing package override.
  695. */
  696. deprecated("Use the overload that accepts a `VersionRange` as 3rd argument")
  697. void removeOverride(PlacementLocation scope_, string package_, Dependency version_spec)
  698. {
  699. version_spec.visit!(
  700. (VersionRange src) => this.removeOverride(scope_, package_, src),
  701. (any) { throw new Exception(format("No override exists for %s %s", package_, version_spec)); },
  702. );
  703. }
  704.  
  705. deprecated(OverrideDepMsg)
  706. void removeOverride(PlacementLocation scope_, string package_, VersionRange src)
  707. {
  708. this.removeOverride_(scope_, package_, src);
  709. }
  710.  
  711. package(dub) void removeOverride_(PlacementLocation scope_, string package_, VersionRange src)
  712. {
  713. Location* rep = &m_repositories[scope_];
  714. foreach (i, ovr; rep.overrides) {
  715. if (ovr.package_ != package_ || ovr.source != src)
  716. continue;
  717. rep.overrides = rep.overrides[0 .. i] ~ rep.overrides[i+1 .. $];
  718. (*rep).writeOverrides();
  719. return;
  720. }
  721. throw new Exception(format("No override exists for %s %s", package_, src));
  722. }
  723.  
  724. deprecated("Use `store(NativePath source, PlacementLocation dest, string name, Version vers)`")
  725. Package storeFetchedPackage(NativePath zip_file_path, Json package_info, NativePath destination)
  726. {
  727. return this.store_(zip_file_path, destination,
  728. PackageName(package_info["name"].get!string),
  729. Version(package_info["version"].get!string));
  730. }
  731.  
  732. /**
  733. * Store a zip file stored at `src` into a managed location `destination`
  734. *
  735. * This will extracts the package supplied as (as a zip file) to the
  736. * `destination` and sets a version field in the package description.
  737. * In the future, we should aim not to alter the package description,
  738. * but this is done for backward compatibility.
  739. *
  740. * Params:
  741. * src = The path to the zip file containing the package
  742. * dest = At which `PlacementLocation` the package should be stored
  743. * name = Name of the package being stored
  744. * vers = Version of the package
  745. *
  746. * Returns:
  747. * The `Package` after it has been loaded.
  748. *
  749. * Throws:
  750. * If the package cannot be loaded / the zip is corrupted / the package
  751. * already exists, etc...
  752. */
  753. deprecated("Use the overload that accepts a `PackageName` instead")
  754. Package store(NativePath src, PlacementLocation dest, string name, Version vers)
  755. {
  756. return this.store(src, dest, PackageName(name), vers);
  757. }
  758.  
  759. /// Ditto
  760. Package store(NativePath src, PlacementLocation dest, in PackageName name,
  761. in Version vers)
  762. {
  763. import dub.internal.vibecompat.core.file;
  764.  
  765. assert(!name.sub.length, "Cannot store a subpackage, use main package instead");
  766. NativePath dstpath = this.getPackagePath(dest, name, vers.toString());
  767. ensureDirectory(dstpath.parentPath());
  768. const lockPath = dstpath.parentPath() ~ ".lock";
  769.  
  770. // possibly wait for other dub instance
  771. import core.time : seconds;
  772. auto lock = lockFile(lockPath.toNativeString(), 30.seconds);
  773. if (dstpath.existsFile()) {
  774. return this.getPackage(name, vers, dest);
  775. }
  776. return this.store_(src, dstpath, name, vers);
  777. }
  778.  
  779. /// Backward-compatibility for deprecated overload, simplify once `storeFetchedPatch`
  780. /// is removed
  781. private Package store_(NativePath src, NativePath destination,
  782. in PackageName name, in Version vers)
  783. {
  784. import dub.internal.vibecompat.core.file;
  785. import std.range : walkLength;
  786.  
  787. logDebug("Placing package '%s' version '%s' to location '%s' from file '%s'",
  788. name, vers, destination.toNativeString(), src.toNativeString());
  789.  
  790. if( existsFile(destination) ){
  791. throw new Exception(format("%s (%s) needs to be removed from '%s' prior placement.",
  792. name, vers, destination));
  793. }
  794.  
  795. // open zip file
  796. ZipArchive archive;
  797. {
  798. logDebug("Opening file %s", src);
  799. archive = new ZipArchive(readFile(src));
  800. }
  801.  
  802. logDebug("Extracting from zip.");
  803.  
  804. // In a GitHub zip, the actual contents are in a sub-folder
  805. alias PSegment = typeof(NativePath.init.head);
  806. PSegment[] zip_prefix;
  807. outer: foreach(ArchiveMember am; archive.directory) {
  808. auto path = NativePath(am.name).bySegment.array;
  809. foreach (fil; packageInfoFiles)
  810. if (path.length == 2 && path[$-1].name == fil.filename) {
  811. zip_prefix = path[0 .. $-1];
  812. break outer;
  813. }
  814. }
  815.  
  816. logDebug("zip root folder: %s", zip_prefix);
  817.  
  818. NativePath getCleanedPath(string fileName) {
  819. auto path = NativePath(fileName);
  820. if (zip_prefix.length && !path.bySegment.startsWith(zip_prefix)) return NativePath.init;
  821. static if (is(typeof(path[0 .. 1]))) return path[zip_prefix.length .. $];
  822. else return NativePath(path.bySegment.array[zip_prefix.length .. $]);
  823. }
  824.  
  825. static void setAttributes(string path, ArchiveMember am)
  826. {
  827. import std.datetime : DosFileTimeToSysTime;
  828.  
  829. auto mtime = DosFileTimeToSysTime(am.time);
  830. setTimes(path, mtime, mtime);
  831. if (auto attrs = am.fileAttributes)
  832. std.file.setAttributes(path, attrs);
  833. }
  834.  
  835. // extract & place
  836. ensureDirectory(destination);
  837. logDebug("Copying all files...");
  838. int countFiles = 0;
  839. foreach(ArchiveMember a; archive.directory) {
  840. auto cleanedPath = getCleanedPath(a.name);
  841. if(cleanedPath.empty) continue;
  842. auto dst_path = destination ~ cleanedPath;
  843.  
  844. logDebug("Creating %s", cleanedPath);
  845. if( dst_path.endsWithSlash ){
  846. ensureDirectory(dst_path);
  847. } else {
  848. ensureDirectory(dst_path.parentPath);
  849. // for symlinks on posix systems, use the symlink function to
  850. // create them. Windows default unzip doesn't handle symlinks,
  851. // so we don't need to worry about it for Windows.
  852. version(Posix) {
  853. import core.sys.posix.sys.stat;
  854. if( S_ISLNK(cast(mode_t)a.fileAttributes) ){
  855. import core.sys.posix.unistd;
  856. // need to convert name and target to zero-terminated string
  857. auto target = toStringz(cast(const(char)[])archive.expand(a));
  858. auto dstFile = toStringz(dst_path.toNativeString());
  859. enforce(symlink(target, dstFile) == 0, "Error creating symlink: " ~ dst_path.toNativeString());
  860. goto symlink_exit;
  861. }
  862. }
  863.  
  864. writeFile(dst_path, archive.expand(a));
  865. setAttributes(dst_path.toNativeString(), a);
  866. symlink_exit:
  867. ++countFiles;
  868. }
  869. }
  870. logDebug("%s file(s) copied.", to!string(countFiles));
  871.  
  872. // overwrite dub.json (this one includes a version field)
  873. auto pack = this.load(destination, NativePath.init, null, vers.toString());
  874.  
  875. if (pack.recipePath.head != defaultPackageFilename)
  876. // Storeinfo saved a default file, this could be different to the file from the zip.
  877. removeFile(pack.recipePath);
  878. pack.storeInfo();
  879. addPackages(this.m_internal.localPackages, pack);
  880. return pack;
  881. }
  882.  
  883. /// Removes the given the package.
  884. void remove(in Package pack)
  885. {
  886. logDebug("Remove %s, version %s, path '%s'", pack.name, pack.version_, pack.path);
  887. enforce(!pack.path.empty, "Cannot remove package "~pack.name~" without a path.");
  888.  
  889. // remove package from repositories' list
  890. bool found = false;
  891. bool removeFrom(Package[] packs, in Package pack) {
  892. auto packPos = countUntil!("a.path == b.path")(packs, pack);
  893. if(packPos != -1) {
  894. packs = .remove(packs, packPos);
  895. return true;
  896. }
  897. return false;
  898. }
  899. foreach(repo; m_repositories) {
  900. if (removeFrom(repo.fromPath, pack)) {
  901. found = true;
  902. break;
  903. }
  904. // Maintain backward compatibility with pre v1.30.0 behavior,
  905. // this is equivalent to remove-local
  906. if (removeFrom(repo.localPackages, pack)) {
  907. found = true;
  908. break;
  909. }
  910. }
  911. if(!found)
  912. found = removeFrom(this.m_internal.localPackages, pack);
  913. enforce(found, "Cannot remove, package not found: '"~ pack.name ~"', path: " ~ to!string(pack.path));
  914.  
  915. logDebug("About to delete root folder for package '%s'.", pack.path);
  916. rmdirRecurse(pack.path.toNativeString());
  917. logInfo("Removed", Color.yellow, "%s %s", pack.name.color(Mode.bold), pack.version_);
  918. }
  919.  
  920. /// Compatibility overload. Use the version without a `force_remove` argument instead.
  921. deprecated("Use `remove(pack)` directly instead, the boolean has no effect")
  922. void remove(in Package pack, bool force_remove)
  923. {
  924. remove(pack);
  925. }
  926.  
  927. Package addLocalPackage(NativePath path, string verName, PlacementLocation type)
  928. {
  929. // As we iterate over `localPackages` we need it to be populated
  930. // In theory we could just populate that specific repository,
  931. // but multiple calls would then become inefficient.
  932. if (!this.m_initialized)
  933. this.refresh();
  934.  
  935. path.endsWithSlash = true;
  936. auto pack = this.load(path);
  937. enforce(pack.name.length, "The package has no name, defined in: " ~ path.toString());
  938. if (verName.length)
  939. pack.version_ = Version(verName);
  940.  
  941. // don't double-add packages
  942. Package[]* packs = &m_repositories[type].localPackages;
  943. foreach (p; *packs) {
  944. if (p.path == path) {
  945. enforce(p.version_ == pack.version_, "Adding the same local package twice with differing versions is not allowed.");
  946. logInfo("Package is already registered: %s (version: %s)", p.name, p.version_);
  947. return p;
  948. }
  949. }
  950.  
  951. addPackages(*packs, pack);
  952.  
  953. this.m_repositories[type].writeLocalPackageList();
  954.  
  955. logInfo("Registered package: %s (version: %s)", pack.name, pack.version_);
  956. return pack;
  957. }
  958.  
  959. void removeLocalPackage(NativePath path, PlacementLocation type)
  960. {
  961. // As we iterate over `localPackages` we need it to be populated
  962. // In theory we could just populate that specific repository,
  963. // but multiple calls would then become inefficient.
  964. if (!this.m_initialized)
  965. this.refresh();
  966.  
  967. path.endsWithSlash = true;
  968. Package[]* packs = &m_repositories[type].localPackages;
  969. size_t[] to_remove;
  970. foreach( i, entry; *packs )
  971. if( entry.path == path )
  972. to_remove ~= i;
  973. enforce(to_remove.length > 0, "No "~type.to!string()~" package found at "~path.toNativeString());
  974.  
  975. string[Version] removed;
  976. foreach (i; to_remove)
  977. removed[(*packs)[i].version_] = (*packs)[i].name;
  978.  
  979. *packs = (*packs).enumerate
  980. .filter!(en => !to_remove.canFind(en.index))
  981. .map!(en => en.value).array;
  982.  
  983. this.m_repositories[type].writeLocalPackageList();
  984.  
  985. foreach(ver, name; removed)
  986. logInfo("Deregistered package: %s (version: %s)", name, ver);
  987. }
  988.  
  989. /// For the given type add another path where packages will be looked up.
  990. void addSearchPath(NativePath path, PlacementLocation type)
  991. {
  992. m_repositories[type].searchPath ~= path;
  993. this.m_repositories[type].writeLocalPackageList();
  994. }
  995.  
  996. /// Removes a search path from the given type.
  997. void removeSearchPath(NativePath path, PlacementLocation type)
  998. {
  999. m_repositories[type].searchPath = m_repositories[type].searchPath.filter!(p => p != path)().array();
  1000. this.m_repositories[type].writeLocalPackageList();
  1001. }
  1002.  
  1003. deprecated("Use `refresh()` without boolean argument(same as `refresh(false)`")
  1004. void refresh(bool refresh)
  1005. {
  1006. if (refresh)
  1007. logDiagnostic("Refreshing local packages (refresh existing: true)...");
  1008. this.refresh_(refresh);
  1009. }
  1010.  
  1011. void refresh()
  1012. {
  1013. this.refresh_(false);
  1014. }
  1015.  
  1016. private void refresh_(bool refresh)
  1017. {
  1018. if (!refresh)
  1019. logDiagnostic("Scanning local packages...");
  1020.  
  1021. foreach (ref repository; this.m_repositories)
  1022. repository.scanLocalPackages(refresh, this);
  1023.  
  1024. this.m_internal.scan(this, refresh);
  1025. foreach (ref repository; this.m_repositories)
  1026. repository.scan(this, refresh);
  1027.  
  1028. foreach (ref repository; this.m_repositories)
  1029. repository.loadOverrides();
  1030. this.m_initialized = true;
  1031. }
  1032.  
  1033. alias Hash = ubyte[];
  1034. /// Generates a hash digest for a given package.
  1035. /// Some files or folders are ignored during the generation (like .dub and
  1036. /// .svn folders)
  1037. Hash hashPackage(Package pack)
  1038. {
  1039. import dub.internal.vibecompat.core.file;
  1040.  
  1041. string[] ignored_directories = [".git", ".dub", ".svn"];
  1042. // something from .dub_ignore or what?
  1043. string[] ignored_files = [];
  1044. SHA256 hash;
  1045. foreach(file; dirEntries(pack.path.toNativeString(), SpanMode.depth)) {
  1046. const isDir = file.isDir;
  1047. if(isDir && ignored_directories.canFind(NativePath(file.name).head.name))
  1048. continue;
  1049. else if(ignored_files.canFind(NativePath(file.name).head.name))
  1050. continue;
  1051.  
  1052. hash.put(cast(ubyte[])NativePath(file.name).head.name);
  1053. if(isDir) {
  1054. logDebug("Hashed directory name %s", NativePath(file.name).head);
  1055. }
  1056. else {
  1057. hash.put(cast(ubyte[]) readFile(NativePath(file.name)));
  1058. logDebug("Hashed file contents from %s", NativePath(file.name).head);
  1059. }
  1060. }
  1061. auto digest = hash.finish();
  1062. logDebug("Project hash: %s", digest);
  1063. return digest[].dup;
  1064. }
  1065.  
  1066. /// Adds the package and scans for sub-packages.
  1067. protected void addPackages(ref Package[] dst_repos, Package pack)
  1068. {
  1069. // Add the main package.
  1070. dst_repos ~= pack;
  1071.  
  1072. // Additionally to the internally defined sub-packages, whose metadata
  1073. // is loaded with the main dub.json, load all externally defined
  1074. // packages after the package is available with all the data.
  1075. foreach (spr; pack.subPackages) {
  1076. Package sp;
  1077.  
  1078. if (spr.path.length) {
  1079. auto p = NativePath(spr.path);
  1080. p.normalize();
  1081. enforce(!p.absolute, "Sub package paths must be sub paths of the parent package.");
  1082. auto path = pack.path ~ p;
  1083. sp = this.load(path, NativePath.init, pack);
  1084. } else sp = new Package(spr.recipe, pack.path, pack);
  1085.  
  1086. // Add the sub-package.
  1087. try {
  1088. dst_repos ~= sp;
  1089. } catch (Exception e) {
  1090. logError("Package '%s': Failed to load sub-package %s: %s", pack.name,
  1091. spr.path.length ? spr.path : spr.recipe.name, e.msg);
  1092. logDiagnostic("Full error: %s", e.toString().sanitize());
  1093. }
  1094. }
  1095. }
  1096.  
  1097. /// Used for dependency injection in `Location`
  1098. protected bool existsDirectory(NativePath path)
  1099. {
  1100. static import dub.internal.vibecompat.core.file;
  1101. return dub.internal.vibecompat.core.file.existsDirectory(path);
  1102. }
  1103.  
  1104. /// Ditto
  1105. protected alias IterateDirDg = int delegate(scope int delegate(ref FileInfo));
  1106.  
  1107. /// Ditto
  1108. protected IterateDirDg iterateDirectory(NativePath path)
  1109. {
  1110. static import dub.internal.vibecompat.core.file;
  1111. return dub.internal.vibecompat.core.file.iterateDirectory(path);
  1112. }
  1113. }
  1114.  
  1115. deprecated(OverrideDepMsg)
  1116. alias PackageOverride = PackageOverride_;
  1117.  
  1118. package(dub) struct PackageOverride_ {
  1119. private alias ResolvedDep = SumType!(NativePath, Version);
  1120. string package_;
  1121. VersionRange source;
  1122. ResolvedDep target;
  1123.  
  1124. deprecated("Use `source` instead")
  1125. @property inout(Dependency) version_ () inout return @safe {
  1126. return Dependency(this.source);
  1127. }
  1128.  
  1129. deprecated("Assign `source` instead")
  1130. @property ref PackageOverride version_ (Dependency v) scope return @safe pure {
  1131. this.source = v.visit!(
  1132. (VersionRange range) => range,
  1133. (any) {
  1134. int a; if (a) return VersionRange.init; // Trick the compiler
  1135. throw new Exception("Cannot use anything else than a `VersionRange` for overrides");
  1136. },
  1137. );
  1138. return this;
  1139. }
  1140.  
  1141. deprecated("Use `target.match` directly instead")
  1142. @property inout(Version) targetVersion () inout return @safe pure nothrow @nogc {
  1143. return this.target.match!(
  1144. (Version v) => v,
  1145. (any) => Version.init,
  1146. );
  1147. }
  1148.  
  1149. deprecated("Assign `target` directly instead")
  1150. @property ref PackageOverride targetVersion (Version v) scope return pure nothrow @nogc {
  1151. this.target = v;
  1152. return this;
  1153. }
  1154.  
  1155. deprecated("Use `target.match` directly instead")
  1156. @property inout(NativePath) targetPath () inout return @safe pure nothrow @nogc {
  1157. return this.target.match!(
  1158. (NativePath v) => v,
  1159. (any) => NativePath.init,
  1160. );
  1161. }
  1162.  
  1163. deprecated("Assign `target` directly instead")
  1164. @property ref PackageOverride targetPath (NativePath v) scope return pure nothrow @nogc {
  1165. this.target = v;
  1166. return this;
  1167. }
  1168.  
  1169. deprecated("Use the overload that accepts a `VersionRange` as 2nd argument")
  1170. this(string package_, Dependency version_, Version target_version)
  1171. {
  1172. this.package_ = package_;
  1173. this.version_ = version_;
  1174. this.target = target_version;
  1175. }
  1176.  
  1177. deprecated("Use the overload that accepts a `VersionRange` as 2nd argument")
  1178. this(string package_, Dependency version_, NativePath target_path)
  1179. {
  1180. this.package_ = package_;
  1181. this.version_ = version_;
  1182. this.target = target_path;
  1183. }
  1184.  
  1185. this(string package_, VersionRange src, Version target)
  1186. {
  1187. this.package_ = package_;
  1188. this.source = src;
  1189. this.target = target;
  1190. }
  1191.  
  1192. this(string package_, VersionRange src, NativePath target)
  1193. {
  1194. this.package_ = package_;
  1195. this.source = src;
  1196. this.target = target;
  1197. }
  1198. }
  1199.  
  1200. deprecated("Use `PlacementLocation` instead")
  1201. enum LocalPackageType : PlacementLocation {
  1202. package_ = PlacementLocation.local,
  1203. user = PlacementLocation.user,
  1204. system = PlacementLocation.system,
  1205. }
  1206.  
  1207. private enum LocalPackagesFilename = "local-packages.json";
  1208. private enum LocalOverridesFilename = "local-overrides.json";
  1209.  
  1210. /**
  1211. * A managed location, with packages, configuration, and overrides
  1212. *
  1213. * There exists three standards locations, listed in `PlacementLocation`.
  1214. * The user one is the default, with the system and local one meeting
  1215. * different needs.
  1216. *
  1217. * Each location has a root, under which the following may be found:
  1218. * - A `packages/` directory, where packages are stored (see `packagePath`);
  1219. * - A `local-packages.json` file, with extra search paths
  1220. * and manually added packages (see `dub add-local`);
  1221. * - A `local-overrides.json` file, with manually added overrides (`dub add-override`);
  1222. *
  1223. * Additionally, each location host a config file,
  1224. * which is not managed by this module, but by dub itself.
  1225. */
  1226. package struct Location {
  1227. /// The absolute path to the root of the location
  1228. NativePath packagePath;
  1229.  
  1230. /// Configured (extra) search paths for this `Location`
  1231. NativePath[] searchPath;
  1232.  
  1233. /**
  1234. * List of manually registered packages at this `Location`
  1235. * and stored in `local-packages.json`
  1236. */
  1237. Package[] localPackages;
  1238.  
  1239. /// List of overrides stored at this `Location`
  1240. PackageOverride_[] overrides;
  1241.  
  1242. /**
  1243. * List of packages stored under `packagePath` and automatically detected
  1244. */
  1245. Package[] fromPath;
  1246.  
  1247. this(NativePath path) @safe pure nothrow @nogc
  1248. {
  1249. this.packagePath = path;
  1250. }
  1251.  
  1252. void loadOverrides()
  1253. {
  1254. import dub.internal.vibecompat.core.file;
  1255.  
  1256. this.overrides = null;
  1257. auto ovrfilepath = this.packagePath ~ LocalOverridesFilename;
  1258. if (existsFile(ovrfilepath)) {
  1259. logWarn("Found local override file: %s", ovrfilepath);
  1260. logWarn(OverrideDepMsg);
  1261. logWarn("Replace with a path-based dependency in your project or a custom cache path");
  1262. foreach (entry; jsonFromFile(ovrfilepath)) {
  1263. PackageOverride_ ovr;
  1264. ovr.package_ = entry["name"].get!string;
  1265. ovr.source = VersionRange.fromString(entry["version"].get!string);
  1266. if (auto pv = "targetVersion" in entry) ovr.target = Version(pv.get!string);
  1267. if (auto pv = "targetPath" in entry) ovr.target = NativePath(pv.get!string);
  1268. this.overrides ~= ovr;
  1269. }
  1270. }
  1271. }
  1272.  
  1273. private void writeOverrides()
  1274. {
  1275. import dub.internal.vibecompat.core.file;
  1276.  
  1277. Json[] newlist;
  1278. foreach (ovr; this.overrides) {
  1279. auto jovr = Json.emptyObject;
  1280. jovr["name"] = ovr.package_;
  1281. jovr["version"] = ovr.source.toString();
  1282. ovr.target.match!(
  1283. (NativePath path) { jovr["targetPath"] = path.toNativeString(); },
  1284. (Version vers) { jovr["targetVersion"] = vers.toString(); },
  1285. );
  1286. newlist ~= jovr;
  1287. }
  1288. auto path = this.packagePath;
  1289. ensureDirectory(path);
  1290. writeJsonFile(path ~ LocalOverridesFilename, Json(newlist));
  1291. }
  1292.  
  1293. private void writeLocalPackageList()
  1294. {
  1295. import dub.internal.vibecompat.core.file;
  1296.  
  1297. Json[] newlist;
  1298. foreach (p; this.searchPath) {
  1299. auto entry = Json.emptyObject;
  1300. entry["name"] = "*";
  1301. entry["path"] = p.toNativeString();
  1302. newlist ~= entry;
  1303. }
  1304.  
  1305. foreach (p; this.localPackages) {
  1306. if (p.parentPackage) continue; // do not store sub packages
  1307. auto entry = Json.emptyObject;
  1308. entry["name"] = p.name;
  1309. entry["version"] = p.version_.toString();
  1310. entry["path"] = p.path.toNativeString();
  1311. newlist ~= entry;
  1312. }
  1313.  
  1314. NativePath path = this.packagePath;
  1315. ensureDirectory(path);
  1316. writeJsonFile(path ~ LocalPackagesFilename, Json(newlist));
  1317. }
  1318.  
  1319. // load locally defined packages
  1320. void scanLocalPackages(bool refresh, PackageManager manager)
  1321. {
  1322. import dub.internal.vibecompat.core.file;
  1323.  
  1324. NativePath list_path = this.packagePath;
  1325. Package[] packs;
  1326. NativePath[] paths;
  1327. try {
  1328. auto local_package_file = list_path ~ LocalPackagesFilename;
  1329. if (!existsFile(local_package_file)) return;
  1330.  
  1331. logDiagnostic("Loading local package map at %s", local_package_file.toNativeString());
  1332. auto packlist = jsonFromFile(local_package_file);
  1333. enforce(packlist.type == Json.Type.array, LocalPackagesFilename ~ " must contain an array.");
  1334. foreach (pentry; packlist) {
  1335. try {
  1336. auto name = pentry["name"].get!string;
  1337. auto path = NativePath(pentry["path"].get!string);
  1338. if (name == "*") {
  1339. paths ~= path;
  1340. } else {
  1341. auto ver = Version(pentry["version"].get!string);
  1342.  
  1343. Package pp;
  1344. if (!refresh) {
  1345. foreach (p; this.localPackages)
  1346. if (p.path == path) {
  1347. pp = p;
  1348. break;
  1349. }
  1350. }
  1351.  
  1352. if (!pp) {
  1353. auto infoFile = Package.findPackageFile(path);
  1354. if (!infoFile.empty) pp = manager.load(path, infoFile);
  1355. else {
  1356. logWarn("Locally registered package %s %s was not found. Please run 'dub remove-local \"%s\"'.",
  1357. name, ver, path.toNativeString());
  1358. // Store a dummy package
  1359. pp = new Package(PackageRecipe(name), path);
  1360. }
  1361. }
  1362.  
  1363. if (pp.name != name)
  1364. logWarn("Local package at %s has different name than %s (%s)", path.toNativeString(), name, pp.name);
  1365. pp.version_ = ver;
  1366. manager.addPackages(packs, pp);
  1367. }
  1368. } catch (Exception e) {
  1369. logWarn("Error adding local package: %s", e.msg);
  1370. }
  1371. }
  1372. } catch (Exception e) {
  1373. logDiagnostic("Loading of local package list at %s failed: %s", list_path.toNativeString(), e.msg);
  1374. }
  1375. this.localPackages = packs;
  1376. this.searchPath = paths;
  1377. }
  1378.  
  1379. /**
  1380. * Scan this location
  1381. */
  1382. void scan(PackageManager mgr, bool refresh)
  1383. {
  1384. // If we're asked to refresh, reload the packages from scratch
  1385. auto existing = refresh ? null : this.fromPath;
  1386. if (this.packagePath !is NativePath.init) {
  1387. // For the internal location, we use `fromPath` to store packages
  1388. // loaded by the user (e.g. the project and its sub-packages),
  1389. // so don't clean it.
  1390. this.fromPath = null;
  1391. }
  1392. foreach (path; this.searchPath)
  1393. this.scanPackageFolder(path, mgr, existing);
  1394. if (this.packagePath !is NativePath.init)
  1395. this.scanPackageFolder(this.packagePath, mgr, existing);
  1396. }
  1397.  
  1398. /**
  1399. * Scan the content of a folder (`packagePath` or in `searchPaths`),
  1400. * and add all packages that were found to this location.
  1401. */
  1402. void scanPackageFolder(NativePath path, PackageManager mgr,
  1403. Package[] existing_packages)
  1404. {
  1405. if (!mgr.existsDirectory(path))
  1406. return;
  1407.  
  1408. void loadInternal (NativePath pack_path, NativePath packageFile)
  1409. {
  1410. Package p;
  1411. try {
  1412. foreach (pp; existing_packages)
  1413. if (pp.path == pack_path) {
  1414. p = pp;
  1415. break;
  1416. }
  1417. if (!p) p = mgr.load(pack_path, packageFile);
  1418. mgr.addPackages(this.fromPath, p);
  1419. } catch (ConfigException exc) {
  1420. // Configy error message already include the path
  1421. logError("Invalid recipe for local package: %S", exc);
  1422. } catch (Exception e) {
  1423. logError("Failed to load package in %s: %s", pack_path, e.msg);
  1424. logDiagnostic("Full error: %s", e.toString().sanitize());
  1425. }
  1426. }
  1427.  
  1428. logDebug("iterating dir %s", path.toNativeString());
  1429. try foreach (pdir; mgr.iterateDirectory(path)) {
  1430. logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
  1431. if (!pdir.isDirectory) continue;
  1432.  
  1433. // Old / flat directory structure, used in non-standard path
  1434. // Packages are stored in $ROOT/$SOMETHING/`
  1435. const pack_path = path ~ (pdir.name ~ "/");
  1436. auto packageFile = Package.findPackageFile(pack_path);
  1437. if (!packageFile.empty) {
  1438. // Deprecated unmanaged directory structure
  1439. logWarn("Package at path '%s' should be under '%s'",
  1440. pack_path.toNativeString().color(Mode.bold),
  1441. (pack_path ~ "$VERSION" ~ pdir.name).toNativeString().color(Mode.bold));
  1442. logWarn("The package will no longer be detected starting from v1.42.0");
  1443. loadInternal(pack_path, packageFile);
  1444. }
  1445.  
  1446. // Managed structure: $ROOT/$NAME/$VERSION/$NAME
  1447. // This is the most common code path
  1448. else if (mgr.isManagedPath(path)) {
  1449. // Iterate over versions of a package
  1450. foreach (versdir; mgr.iterateDirectory(pack_path)) {
  1451. if (!versdir.isDirectory) continue;
  1452. auto vers_path = pack_path ~ versdir.name ~ (pdir.name ~ "/");
  1453. if (!mgr.existsDirectory(vers_path)) continue;
  1454. packageFile = Package.findPackageFile(vers_path);
  1455. loadInternal(vers_path, packageFile);
  1456. }
  1457. }
  1458. }
  1459. catch (Exception e)
  1460. logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString());
  1461. }
  1462.  
  1463. /**
  1464. * Looks up already-loaded packages at a specific version
  1465. *
  1466. * Looks up a package according to this `Location`'s priority,
  1467. * that is, packages from the search path and local packages
  1468. * have the highest priority.
  1469. *
  1470. * Params:
  1471. * name = The full name of the package to look up
  1472. * ver = The version to look up
  1473. *
  1474. * Returns:
  1475. * A `Package` if one was found, `null` if none exists.
  1476. */
  1477. inout(Package) lookup(in PackageName name, in Version ver) inout {
  1478. foreach (pkg; this.localPackages)
  1479. if (pkg.name == name.toString() &&
  1480. pkg.version_.matches(ver, VersionMatchMode.standard))
  1481. return pkg;
  1482. foreach (pkg; this.fromPath) {
  1483. auto pvm = this.isManaged(pkg.basePackage.path) ?
  1484. VersionMatchMode.strict : VersionMatchMode.standard;
  1485. if (pkg.name == name.toString() && pkg.version_.matches(ver, pvm))
  1486. return pkg;
  1487. }
  1488. return null;
  1489. }
  1490.  
  1491. /**
  1492. * Looks up a package, first in the list of loaded packages,
  1493. * then directly on the file system.
  1494. *
  1495. * This function allows for lazy loading of packages, without needing to
  1496. * first scan all the available locations (as `scan` does).
  1497. *
  1498. * Params:
  1499. * name = The full name of the package to look up
  1500. * vers = The version the package must match
  1501. * mgr = The `PackageManager` to use for adding packages
  1502. *
  1503. * Returns:
  1504. * A `Package` if one was found, `null` if none exists.
  1505. */
  1506. Package load (in PackageName name, Version vers, PackageManager mgr)
  1507. {
  1508. if (auto pkg = this.lookup(name, vers))
  1509. return pkg;
  1510.  
  1511. string versStr = vers.toString();
  1512. const path = this.getPackagePath(name, versStr);
  1513. if (!mgr.existsDirectory(path))
  1514. return null;
  1515.  
  1516. logDiagnostic("Lazily loading package %s:%s from %s", name.main, vers, path);
  1517. auto p = mgr.load(path);
  1518. enforce(
  1519. p.version_ == vers,
  1520. format("Package %s located in %s has a different version than its path: Got %s, expected %s",
  1521. name, path, p.version_, vers));
  1522. mgr.addPackages(this.fromPath, p);
  1523. return p;
  1524. }
  1525.  
  1526. /**
  1527. * Get the final destination a specific package needs to be stored in.
  1528. *
  1529. * Note that there needs to be an extra level for libraries like `ae`
  1530. * which expects their containing folder to have an exact name and use
  1531. * `importPath "../"`.
  1532. *
  1533. * Hence the final format returned is `$BASE/$NAME/$VERSION/$NAME`,
  1534. * `$BASE` is `this.packagePath`.
  1535. *
  1536. * Params:
  1537. * name = The package name - if the name is that of a subpackage,
  1538. * only the path to the main package is returned, as the
  1539. * subpackage path can only be known after reading the recipe.
  1540. * vers = A version string. Typed as a string because git hashes
  1541. * can be used with this function.
  1542. *
  1543. * Returns:
  1544. * An absolute `NativePath` nested in this location.
  1545. */
  1546. NativePath getPackagePath (in PackageName name, string vers)
  1547. {
  1548. NativePath result = this.packagePath ~ name.main.toString() ~ vers ~
  1549. name.main.toString();
  1550. result.endsWithSlash = true;
  1551. return result;
  1552. }
  1553.  
  1554. /// Determines if a specific path is within a DUB managed Location.
  1555. bool isManaged(NativePath path) const {
  1556. return path.startsWith(this.packagePath);
  1557. }
  1558. }
  1559.  
  1560. private immutable string OverrideDepMsg =
  1561. "Overrides are deprecated as they are redundant with more fine-grained approaches";