Newer
Older
dub_jkp / source / dub / dependencyresolver.d
  1. /**
  2. Dependency configuration/version resolution algorithm.
  3.  
  4. Copyright: © 2014 rejectedsoftware e.K.
  5. License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
  6. Authors: Sönke Ludwig
  7. */
  8. module dub.dependencyresolver;
  9.  
  10. import dub.dependency;
  11. import dub.internal.vibecompat.core.log;
  12.  
  13. import std.algorithm : all, canFind, filter, sort;
  14. import std.array : appender, array;
  15. import std.conv : to;
  16. import std.exception : enforce;
  17. import std.string : format, indexOf, lastIndexOf;
  18.  
  19.  
  20. class DependencyResolver(CONFIGS, CONFIG) {
  21. static struct TreeNodes {
  22. string pack;
  23. CONFIGS configs;
  24. DependencyType depType = DependencyType.required;
  25.  
  26. hash_t toHash() const nothrow @trusted {
  27. size_t ret = typeid(string).getHash(&pack);
  28. ret ^= typeid(CONFIGS).getHash(&configs);
  29. return ret;
  30. }
  31. bool opEqual(in ref TreeNodes other) const { return pack == other.pack && configs == other.configs; }
  32. int opCmp(in ref TreeNodes other) const {
  33. if (pack != other.pack) return pack < other.pack ? -1 : 1;
  34. if (configs != other.configs) return configs < other.configs ? -1 : 1;
  35. return 0;
  36. }
  37. }
  38.  
  39. static struct TreeNode {
  40. string pack;
  41. CONFIG config;
  42.  
  43. hash_t toHash() const nothrow @trusted {
  44. size_t ret = typeid(string).getHash(&pack);
  45. ret ^= typeid(CONFIG).getHash(&config);
  46. return ret;
  47. }
  48. bool opEqual(in ref TreeNode other) const { return pack == other.pack && config == other.config; }
  49. int opCmp(in ref TreeNode other) const {
  50. if (pack != other.pack) return pack < other.pack ? -1 : 1;
  51. if (config != other.config) return config < other.config ? -1 : 1;
  52. return 0;
  53. }
  54. }
  55.  
  56. CONFIG[string] resolve(TreeNode root, bool throw_on_failure = true)
  57. {
  58. auto root_base_pack = rootPackage(root.pack);
  59.  
  60. // find all possible configurations of each possible dependency
  61. size_t[string] package_indices;
  62. string[size_t] package_names;
  63. CONFIG[][] all_configs;
  64. bool[string] required_deps;
  65. bool[TreeNode] visited;
  66. void findConfigsRec(TreeNode parent, bool parent_unique)
  67. {
  68. if (parent in visited) return;
  69. visited[parent] = true;
  70.  
  71. foreach (ch; getChildren(parent)) {
  72. auto basepack = rootPackage(ch.pack);
  73. auto pidx = all_configs.length;
  74.  
  75. if (ch.depType == DependencyType.required) required_deps[ch.pack] = true;
  76.  
  77. CONFIG[] configs;
  78. if (auto pi = basepack in package_indices) {
  79. pidx = *pi;
  80. configs = all_configs[*pi];
  81. } else {
  82. if (basepack == root_base_pack) configs = [root.config];
  83. else configs = getAllConfigs(basepack);
  84. all_configs ~= configs;
  85. package_indices[basepack] = pidx;
  86. package_names[pidx] = basepack;
  87. }
  88.  
  89. configs = getSpecificConfigs(ch) ~ configs;
  90.  
  91. // eliminate configurations from which we know that they can't satisfy
  92. // the uniquely defined root dependencies (==version or ~branch style dependencies)
  93. if (parent_unique) configs = configs.filter!(c => matches(ch.configs, c)).array;
  94.  
  95. all_configs[pidx] = configs;
  96.  
  97. foreach (v; configs)
  98. findConfigsRec(TreeNode(ch.pack, v), parent_unique && configs.length == 1);
  99. }
  100. }
  101. findConfigsRec(root, true);
  102.  
  103. // append an invalid configuration to denote an unchosen dependency
  104. // this is used to properly support optional dependencies (when
  105. // getChildren() returns no configurations for an optional dependency,
  106. // but getAllConfigs() has already provided an existing list of configs)
  107. foreach (i, ref cfgs; all_configs)
  108. if (cfgs.length == 0 || package_names[i] !in required_deps)
  109. cfgs = cfgs ~ CONFIG.invalid;
  110.  
  111. logDebug("Configurations used for dependency resolution:");
  112. foreach (n, i; package_indices) logDebug(" %s (%s): %s", n, i, all_configs[i]);
  113.  
  114. auto config_indices = new size_t[all_configs.length];
  115. config_indices[] = 0;
  116.  
  117. string last_error;
  118.  
  119. visited = null;
  120. sizediff_t validateConfigs(TreeNode parent, ref string error)
  121. {
  122. import std.algorithm : max;
  123.  
  124. if (parent in visited) return -1;
  125.  
  126. visited[parent] = true;
  127. sizediff_t maxcpi = -1;
  128. sizediff_t parentidx = package_indices.get(rootPackage(parent.pack), -1);
  129. auto parentbase = rootPackage(parent.pack);
  130.  
  131. // loop over all dependencies
  132. foreach (ch; getChildren(parent)) {
  133. auto basepack = rootPackage(ch.pack);
  134. assert(basepack in package_indices, format("%s not in packages %s", basepack, package_indices));
  135.  
  136. // get the current config/version of the current dependency
  137. sizediff_t childidx = package_indices[basepack];
  138. if (all_configs[childidx] == [CONFIG.invalid]) {
  139. // ignore invalid optional dependencies
  140. if (ch.depType != DependencyType.required)
  141. continue;
  142.  
  143. enforce(parentbase != root_base_pack, format("Root package %s contains reference to invalid package %s %s", parent.pack, ch.pack, ch.configs));
  144. // choose another parent config to avoid the invalid child
  145. if (parentidx > maxcpi) {
  146. error = format("Package %s contains invalid dependency %s", parent.pack, ch.pack);
  147. logDiagnostic("%s (ci=%s)", error, parentidx);
  148. maxcpi = parentidx;
  149. }
  150. } else {
  151. auto config = all_configs[childidx][config_indices[childidx]];
  152. auto chnode = TreeNode(ch.pack, config);
  153.  
  154. if (config == CONFIG.invalid || !matches(ch.configs, config)) {
  155. // ignore missing optional dependencies
  156. if (config == CONFIG.invalid && ch.depType != DependencyType.required)
  157. continue;
  158.  
  159. // if we are at the root level, we can safely skip the maxcpi computation and instead choose another childidx config
  160. if (parentbase == root_base_pack) {
  161. error = format("No match for dependency %s %s of %s", ch.pack, ch.configs, parent.pack);
  162. return childidx;
  163. }
  164.  
  165. if (childidx > maxcpi) {
  166. maxcpi = max(childidx, parentidx);
  167. error = format("Dependency %s -> %s %s mismatches with selected version %s", parent.pack, ch.pack, ch.configs, config);
  168. logDebug("%s (ci=%s)", error, maxcpi);
  169. }
  170.  
  171. // we know that either the child or the parent needs to be switched
  172. // to another configuration, no need to continue with other children
  173. if (config == CONFIG.invalid) break;
  174. }
  175.  
  176. maxcpi = max(maxcpi, validateConfigs(chnode, error));
  177. }
  178. }
  179. return maxcpi;
  180. }
  181.  
  182. string first_error;
  183.  
  184. while (true) {
  185. // check if the current combination of configurations works out
  186. visited = null;
  187. string error;
  188. auto conflict_index = validateConfigs(root, error);
  189. if (first_error !is null) first_error = error;
  190.  
  191. // print out current iteration state
  192. logDebug("Interation (ci=%s) %s", conflict_index, {
  193. import std.array : join;
  194. auto cs = new string[all_configs.length];
  195. foreach (p, i; package_indices) {
  196. if (all_configs[i].length)
  197. cs[i] = p~" "~all_configs[i][config_indices[i]].to!string~(i >= 0 && i >= conflict_index ? " (C)" : "");
  198. else cs[i] = p ~ " [no config]";
  199. }
  200. return cs.join(", ");
  201. }());
  202.  
  203. if (conflict_index < 0) {
  204. CONFIG[string] ret;
  205. foreach (p, i; package_indices)
  206. if (all_configs[i].length) {
  207. auto cfg = all_configs[i][config_indices[i]];
  208. if (cfg != CONFIG.invalid) ret[p] = cfg;
  209. }
  210. purgeOptionalDependencies(root, ret);
  211. return ret;
  212. }
  213.  
  214. // find the next combination of configurations
  215. foreach_reverse (pi, ref i; config_indices) {
  216. if (pi > conflict_index) i = 0;
  217. else if (++i >= all_configs[pi].length) i = 0;
  218. else break;
  219. }
  220. if (config_indices.all!"a==0") {
  221. if (throw_on_failure) throw new Exception("Could not find a valid dependency tree configuration: "~first_error);
  222. else return null;
  223. }
  224. }
  225. }
  226.  
  227. protected abstract CONFIG[] getAllConfigs(string pack);
  228. protected abstract CONFIG[] getSpecificConfigs(TreeNodes nodes);
  229. protected abstract TreeNodes[] getChildren(TreeNode node);
  230. protected abstract bool matches(CONFIGS configs, CONFIG config);
  231.  
  232. private void purgeOptionalDependencies(TreeNode root, ref CONFIG[string] configs)
  233. {
  234. bool[string] required;
  235.  
  236. void markRecursively(TreeNode node)
  237. {
  238. if (node.pack in required) return;
  239. required[node.pack] = true;
  240. foreach (dep; getChildren(node).filter!(dep => dep.depType != DependencyType.optional))
  241. if (auto dp = rootPackage(dep.pack) in configs)
  242. markRecursively(TreeNode(dep.pack, *dp));
  243. }
  244.  
  245. // recursively mark all required dependencies of the concrete dependency tree
  246. markRecursively(root);
  247.  
  248. // remove all un-marked configurations
  249. foreach (p; configs.keys.dup)
  250. if (p !in required)
  251. configs.remove(p);
  252. }
  253. }
  254.  
  255. enum DependencyType {
  256. required,
  257. optionalDefault,
  258. optional
  259. }
  260.  
  261. private string rootPackage(string p)
  262. {
  263. auto idx = indexOf(p, ":");
  264. if (idx < 0) return p;
  265. return p[0 .. idx];
  266. }
  267.  
  268.  
  269. unittest {
  270. static struct IntConfig {
  271. int value;
  272. alias value this;
  273. enum invalid = IntConfig(-1);
  274. }
  275. static IntConfig ic(int v) { return IntConfig(v); }
  276. static struct IntConfigs {
  277. IntConfig[] configs;
  278. alias configs this;
  279. }
  280. static IntConfigs ics(IntConfig[] cfgs) { return IntConfigs(cfgs); }
  281.  
  282. static class TestResolver : DependencyResolver!(IntConfigs, IntConfig) {
  283. private TreeNodes[][string] m_children;
  284. this(TreeNodes[][string] children) { m_children = children; }
  285. protected override IntConfig[] getAllConfigs(string pack) {
  286. auto ret = appender!(IntConfig[]);
  287. foreach (p; m_children.byKey) {
  288. if (p.length <= pack.length+1) continue;
  289. if (p[0 .. pack.length] != pack || p[pack.length] != ':') continue;
  290. auto didx = p.lastIndexOf(':');
  291. ret ~= ic(p[didx+1 .. $].to!uint);
  292. }
  293. ret.data.sort!"a>b"();
  294. return ret.data;
  295. }
  296. protected override IntConfig[] getSpecificConfigs(TreeNodes nodes) { return null; }
  297. protected override TreeNodes[] getChildren(TreeNode node) { return m_children.get(node.pack ~ ":" ~ node.config.to!string(), null); }
  298. protected override bool matches(IntConfigs configs, IntConfig config) { return configs.canFind(config); }
  299. }
  300.  
  301. // properly back up if conflicts are detected along the way (d:2 vs d:1)
  302. with (TestResolver) {
  303. auto res = new TestResolver([
  304. "a:0": [TreeNodes("b", ics([ic(2), ic(1)])), TreeNodes("d", ics([ic(1)])), TreeNodes("e", ics([ic(2), ic(1)]))],
  305. "b:1": [TreeNodes("c", ics([ic(2), ic(1)])), TreeNodes("d", ics([ic(1)]))],
  306. "b:2": [TreeNodes("c", ics([ic(3), ic(2)])), TreeNodes("d", ics([ic(2), ic(1)]))],
  307. "c:1": [], "c:2": [], "c:3": [],
  308. "d:1": [], "d:2": [],
  309. "e:1": [], "e:2": [],
  310. ]);
  311. assert(res.resolve(TreeNode("a", ic(0))) == ["b":ic(2), "c":ic(3), "d":ic(1), "e":ic(2)], format("%s", res.resolve(TreeNode("a", ic(0)))));
  312. }
  313.  
  314. // handle cyclic dependencies gracefully
  315. with (TestResolver) {
  316. auto res = new TestResolver([
  317. "a:0": [TreeNodes("b", ics([ic(1)]))],
  318. "b:1": [TreeNodes("b", ics([ic(1)]))]
  319. ]);
  320. assert(res.resolve(TreeNode("a", ic(0))) == ["b":ic(1)]);
  321. }
  322.  
  323. // don't choose optional dependencies by default
  324. with (TestResolver) {
  325. auto res = new TestResolver([
  326. "a:0": [TreeNodes("b", ics([ic(1)]), DependencyType.optional)],
  327. "b:1": []
  328. ]);
  329. assert(res.resolve(TreeNode("a", ic(0))).length == 0, to!string(res.resolve(TreeNode("a", ic(0)))));
  330. }
  331.  
  332. // choose default optional dependencies by default
  333. with (TestResolver) {
  334. auto res = new TestResolver([
  335. "a:0": [TreeNodes("b", ics([ic(1)]), DependencyType.optionalDefault)],
  336. "b:1": []
  337. ]);
  338. assert(res.resolve(TreeNode("a", ic(0))) == ["b":ic(1)], to!string(res.resolve(TreeNode("a", ic(0)))));
  339. }
  340.  
  341. // choose optional dependency if non-optional within the dependency tree
  342. with (TestResolver) {
  343. auto res = new TestResolver([
  344. "a:0": [TreeNodes("b", ics([ic(1)]), DependencyType.optional), TreeNodes("c", ics([ic(1)]))],
  345. "b:1": [],
  346. "c:1": [TreeNodes("b", ics([ic(1)]))]
  347. ]);
  348. assert(res.resolve(TreeNode("a", ic(0))) == ["b":ic(1), "c":ic(1)], to!string(res.resolve(TreeNode("a", ic(0)))));
  349. }
  350.  
  351. // don't choose optional dependency if non-optional outside of final dependency tree
  352. with (TestResolver) {
  353. auto res = new TestResolver([
  354. "a:0": [TreeNodes("b", ics([ic(1)]), DependencyType.optional)],
  355. "b:1": [],
  356. "preset:0": [TreeNodes("b", ics([ic(1)]))]
  357. ]);
  358. assert(res.resolve(TreeNode("a", ic(0))).length == 0, to!string(res.resolve(TreeNode("a", ic(0)))));
  359. }
  360. }