From 7cbd0a03019b542cd2827e32910598b41e09f26d Mon Sep 17 00:00:00 2001 From: Igor Gnatenko Date: Wed, 28 Feb 2018 19:42:48 +0100 Subject: [PATCH] mmd_resolver: rework solvables generation Now we have good separation between runtime and buildtime dependencies: If mmd has context, it is built artefact which supposed to have only one dependency entry, we create solvable with real arch for it. Otherwise we create multiple solvables with "src" arch. In "src" solvables, "context" is a number which represents index in dependencies array. Stream is not included anymore in "evr" because it's incomparable. Version is not included anymore in "name" because it's there just for comparison, nothing more (it's not different module-stream). As a side, add_requires/add_conflicts/add_provides were replaced with non-deprecated add_deparray method. With all this, we are ready to handle MMDs which have multiple elements in dependencies (modulemd v2). Signed-off-by: Igor Gnatenko --- module_build_service/mmd_resolver.py | 120 +++++++++------- tests/test_mmd_resolver.py | 202 ++++++++++++++++++--------- 2 files changed, 203 insertions(+), 119 deletions(-) diff --git a/module_build_service/mmd_resolver.py b/module_build_service/mmd_resolver.py index 37de0f66..eaaa4fce 100644 --- a/module_build_service/mmd_resolver.py +++ b/module_build_service/mmd_resolver.py @@ -115,68 +115,75 @@ class MMDResolver(object): self.build_repo = self.pool.add_repo("build") self.available_repo = self.pool.add_repo("available") - self.alternatives_whitelist = set() + def add_modules(self, mmd): + n, s, v, c = mmd.get_name(), mmd.get_stream(), mmd.get_version(), mmd.get_context() - def _create_solvable(self, repo, mmd): - """ - Creates libsolv Solvable object in repo `repo` based on the Modulemd - metadata `mmd`. + pool = self.pool - This fills in all the provides/requires/conflicts of Solvable. + solvables = [] + if c is not None: + # Built module + deps = mmd.get_dependencies() + if len(deps) > 1: + assert False, "Multiple dependencies for runtime" - :rtype: solv.Solvable - :return: Solvable object. - """ - solvable = repo.add_solvable() - solvable.name = "%s:%s:%d:%s" % (mmd.get_name(), mmd.get_stream(), - mmd.get_version(), mmd.get_context()) - solvable.evr = "%s-%d" % (mmd.get_stream(), mmd.get_version()) - solvable.arch = "x86_64" + # $n:$s:$c-$v.$a + solvable = self.available_repo.add_solvable() + solvable.name = "%s:%s:%d:%s" % (n, s, v, c) + solvable.evr = str(v) + # TODO: replace with real arch + solvable.arch = "x86_64" - # Provides - solvable.add_provides( - self.pool.Dep("module(%s)" % mmd.get_name()).Rel( - solv.REL_EQ, self.pool.Dep(solvable.evr))) + # Prv: module($n) + solvable.add_deparray(solv.SOLVABLE_PROVIDES, + pool.Dep("module(%s)" % n)) + # Prv: module($n:$s) = $v + solvable.add_deparray(solv.SOLVABLE_PROVIDES, + pool.Dep("module(%s:%s)" % (n, s)).Rel( + solv.REL_EQ, pool.Dep(str(v)))) - # Requires - for deps in mmd.get_dependencies(): - for name, streams in deps.get_requires().items(): - requires = None - for stream in streams.get(): - require = self.pool.Dep("module(%s)" % name) - require = require.Rel(solv.REL_EQ, self.pool.Dep(stream)) - if requires: - requires = requires.Rel(solv.REL_OR, require) - else: - requires = require - solvable.add_requires(requires) + if deps: + # Req: (module($on1:$os1) OR module($on2:$os2) OR …) + for name, streams in deps[0].get_requires().items(): + requires = None + for stream in streams.get(): + require = pool.Dep("module(%s:%s)" % (name, stream)) + if requires is not None: + requires = requires.Rel(solv.REL_OR, require) + else: + requires = require + solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires) - # Build-requires in case we are in build_repo. - if repo == self.build_repo: - solvable.arch = "src" - for deps in mmd.get_dependencies(): + # Con: module($n) + solvable.add_deparray(solv.SOLVABLE_CONFLICTS, pool.Dep("module(%s)" % n)) + + solvables.append(solvable) + else: + # Input module + # Context means two things: + # * Unique identifier + # * Offset for the dependency which was used + for c, deps in enumerate(mmd.get_dependencies()): + # $n:$s:$c-$v.src + solvable = self.build_repo.add_solvable() + solvable.name = "%s:%s:%s:%d" % (n, s, v, c) + solvable.evr = str(v) + solvable.arch = "src" + + # Req: (module($on1:$os1) OR module($on2:$os2) OR …) for name, streams in deps.get_buildrequires().items(): requires = None for stream in streams.get(): - require = self.pool.Dep("module(%s)" % name) - require = require.Rel(solv.REL_EQ, self.pool.Dep(stream)) + require = pool.Dep("module(%s:%s)" % (name, stream)) if requires: requires = requires.Rel(solv.REL_OR, require) else: requires = require - solvable.add_requires(requires) - self.alternatives_whitelist.add(name) + solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires) - # Conflicts - solvable.add_conflicts(self.pool.Dep("module(%s)" % mmd.get_name())) + solvables.append(solvable) - return solvable - - def add_available_module(self, mmd): - """ - Adds module available for dependency solving. - """ - self._create_solvable(self.available_repo, mmd) + return solvables def solve(self, mmd): """ @@ -187,13 +194,22 @@ class MMDResolver(object): :return: set of frozensets of n:s:v:c of modules which satisfied the dependency solving. """ - solvable = self._create_solvable(self.build_repo, mmd) + solvables = self.add_modules(mmd) + if not solvables: + raise ValueError("No module(s) found for resolving") self.pool.createwhatprovides() - new_job = self.pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE, solvable.id) + # XXX: Using SOLVABLE_ONE_OF should be faster & more convenient. + # There must be a bug in _gather_alternatives(), possibly when processing l1 alternatives. + # Use pool.towhatprovides() to combine solvables. + + alternatives = [] jobs = self.pool.getpooljobs() - self.pool.setpooljobs(jobs + [new_job]) - alternatives = _gather_alternatives(self.pool) + for s in solvables: + new_job = self.pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE, s.id) + self.pool.setpooljobs(jobs + [new_job]) + alternatives.extend(_gather_alternatives(self.pool)) self.pool.setpooljobs(jobs) - return set(frozenset(s.name for s in trans if s != solvable) for trans in alternatives) + return set(frozenset("%s:%s" % (s.name, s.arch) for s in trans) + for trans in alternatives) diff --git a/tests/test_mmd_resolver.py b/tests/test_mmd_resolver.py index e970508f..e5b43a37 100644 --- a/tests/test_mmd_resolver.py +++ b/tests/test_mmd_resolver.py @@ -24,7 +24,7 @@ # Igor Gnatenko import gi -gi.require_version('Modulemd', '1.0') # noqa +gi.require_version("Modulemd", "1.0") # noqa from gi.repository import Modulemd from module_build_service.mmd_resolver import MMDResolver @@ -38,14 +38,12 @@ class TestMMDResolver: def teardown_method(self, test_method): pass - def _make_mmd(self, nsvc, requires, build_requires): - name, stream, version, context = nsvc.split(":") + def _make_mmd(self, nsvc, requires): + name, stream, version = nsvc.split(":", 2) mmd = Modulemd.Module() mmd.set_mdversion(2) mmd.set_name(name) mmd.set_stream(stream) - mmd.set_version(int(version)) - mmd.set_context(context) mmd.set_summary("foo") mmd.set_description("foo") licenses = Modulemd.SimpleSet() @@ -53,98 +51,168 @@ class TestMMDResolver: mmd.set_module_licenses(licenses) deps = Modulemd.Dependencies() + if ":" in version: + version, context = version.split(":") + mmd.set_context(context) + add_requires = deps.add_requires + else: + add_requires = deps.add_buildrequires + mmd.set_version(int(version)) + for req_name, req_streams in requires.items(): - deps.add_requires(req_name, req_streams) - for req_name, req_streams in build_requires.items(): - deps.add_buildrequires(req_name, req_streams) + add_requires(req_name, req_streams) mmd.set_dependencies((deps, )) + return mmd def test_solve_tree(self): - mmds = [] - mmds.append(self._make_mmd("app:1:0:c1", {}, {"gtk": ["1", "2"]})) - mmds.append(self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}, {})) - mmds.append(self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}, {})) - mmds.append(self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}, {})) - mmds.append(self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}, {})) - mmds.append(self._make_mmd("font:a:0:c6", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("font:a:0:c7", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("font:b:0:c8", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("font:b:0:c9", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("platform:f28:0:c10", {}, {})) - mmds.append(self._make_mmd("platform:f29:0:c11", {}, {})) + mmds = [ + self._make_mmd("app:1:0", {"gtk": ["1", "2"]}), + self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}), + self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}), + self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}), + self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}), + self._make_mmd("font:a:0:c6", {"platform": ["f28"]}), + self._make_mmd("font:a:0:c7", {"platform": ["f29"]}), + self._make_mmd("font:b:0:c8", {"platform": ["f28"]}), + self._make_mmd("font:b:0:c9", {"platform": ["f29"]}), + self._make_mmd("platform:f28:0:c10", {}), + self._make_mmd("platform:f29:0:c11", {}), + ] for mmd in mmds[1:]: - self.mmd_resolver.add_available_module(mmd) + self.mmd_resolver.add_modules(mmd) expanded = self.mmd_resolver.solve(mmds[0]) expected = set([ - frozenset(["gtk:1:0:c2", "font:a:0:c6", "platform:f28:0:c10"]), - frozenset(["gtk:1:0:c2", "font:b:0:c8", "platform:f28:0:c10"]), - frozenset(["gtk:1:0:c3", "font:a:0:c7", "platform:f29:0:c11"]), - frozenset(["gtk:1:0:c3", "font:b:0:c9", "platform:f29:0:c11"]), - frozenset(["gtk:2:0:c4", "font:a:0:c6", "platform:f28:0:c10"]), - frozenset(["gtk:2:0:c4", "font:b:0:c8", "platform:f28:0:c10"]), - frozenset(["gtk:2:0:c5", "font:a:0:c7", "platform:f29:0:c11"]), - frozenset(["gtk:2:0:c5", "font:b:0:c9", "platform:f29:0:c11"]), + frozenset(["app:1:0:0:src", + "font:a:0:c6:x86_64", + "gtk:1:0:c2:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "font:a:0:c7:x86_64", + "gtk:1:0:c3:x86_64", + "platform:f29:0:c11:x86_64"]), + frozenset(["app:1:0:0:src", + "font:b:0:c8:x86_64", + "gtk:1:0:c2:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "font:b:0:c9:x86_64", + "gtk:1:0:c3:x86_64", + "platform:f29:0:c11:x86_64"]), + frozenset(["app:1:0:0:src", + "font:a:0:c6:x86_64", + "gtk:2:0:c4:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "font:b:0:c9:x86_64", + "gtk:2:0:c5:x86_64", + "platform:f29:0:c11:x86_64"]), + frozenset(["app:1:0:0:src", + "font:b:0:c8:x86_64", + "gtk:2:0:c4:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "font:a:0:c7:x86_64", + "gtk:2:0:c5:x86_64", + "platform:f29:0:c11:x86_64"]), ]) assert expanded == expected def test_solve_tree_buildrequire_platform(self): - mmds = [] - mmds.append(self._make_mmd("app:1:0:c1", {}, {"gtk": ["1", "2"], "platform": ["f28"]})) - mmds.append(self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}, {})) - mmds.append(self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}, {})) - mmds.append(self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}, {})) - mmds.append(self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}, {})) - mmds.append(self._make_mmd("font:a:0:c6", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("font:a:0:c7", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("font:b:0:c8", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("font:b:0:c9", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("platform:f28:0:c10", {}, {})) - mmds.append(self._make_mmd("platform:f29:0:c11", {}, {})) + mmds = [ + self._make_mmd("app:1:0", {"gtk": ["1", "2"], "platform": ["f28"]}), + self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}), + self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}), + self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}), + self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}), + self._make_mmd("font:a:0:c6", {"platform": ["f28"]}), + self._make_mmd("font:a:0:c7", {"platform": ["f29"]}), + self._make_mmd("font:b:0:c8", {"platform": ["f28"]}), + self._make_mmd("font:b:0:c9", {"platform": ["f29"]}), + self._make_mmd("platform:f28:0:c10", {}), + self._make_mmd("platform:f29:0:c11", {}), + ] for mmd in mmds[1:]: - self.mmd_resolver.add_available_module(mmd) + self.mmd_resolver.add_modules(mmd) expanded = self.mmd_resolver.solve(mmds[0]) expected = set([ - frozenset(["gtk:1:0:c2", "platform:f28:0:c10", "font:a:0:c6"]), - frozenset(["gtk:1:0:c2", "platform:f28:0:c10", "font:b:0:c8"]), - frozenset(["gtk:2:0:c4", "platform:f28:0:c10", "font:a:0:c6"]), - frozenset(["gtk:2:0:c4", "platform:f28:0:c10", "font:b:0:c8"]), + frozenset(["app:1:0:0:src", + "font:a:0:c6:x86_64", + "gtk:2:0:c4:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "font:b:0:c8:x86_64", + "gtk:1:0:c2:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "font:a:0:c6:x86_64", + "gtk:1:0:c2:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "font:b:0:c8:x86_64", + "gtk:2:0:c4:x86_64", + "platform:f28:0:c10:x86_64"]), ]) assert expanded == expected def test_solve_tree_multiple_build_requires(self): - mmds = [] - mmds.append(self._make_mmd("app:1:0:c1", {}, {"gtk": ["1", "2"], "foo": ["1", "2"]})) - mmds.append(self._make_mmd("gtk:1:0:c2", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("gtk:1:0:c3", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("gtk:2:0:c4", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("gtk:2:0:c5", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("foo:1:0:c2", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("foo:1:0:c3", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("foo:2:0:c4", {"platform": ["f28"]}, {})) - mmds.append(self._make_mmd("foo:2:0:c5", {"platform": ["f29"]}, {})) - mmds.append(self._make_mmd("platform:f28:0:c10", {}, {})) - mmds.append(self._make_mmd("platform:f29:0:c11", {}, {})) + mmds = [ + self._make_mmd("app:1:0", {"gtk": ["1", "2"], "foo": ["1", "2"]}), + self._make_mmd("gtk:1:0:c2", {"platform": ["f28"]}), + self._make_mmd("gtk:1:0:c3", {"platform": ["f29"]}), + self._make_mmd("gtk:2:0:c4", {"platform": ["f28"]}), + self._make_mmd("gtk:2:0:c5", {"platform": ["f29"]}), + self._make_mmd("foo:1:0:c2", {"platform": ["f28"]}), + self._make_mmd("foo:1:0:c3", {"platform": ["f29"]}), + self._make_mmd("foo:2:0:c4", {"platform": ["f28"]}), + self._make_mmd("foo:2:0:c5", {"platform": ["f29"]}), + self._make_mmd("platform:f28:0:c10", {}), + self._make_mmd("platform:f29:0:c11", {}), + ] for mmd in mmds[1:]: - self.mmd_resolver.add_available_module(mmd) + self.mmd_resolver.add_modules(mmd) expanded = self.mmd_resolver.solve(mmds[0]) expected = set([ - frozenset(['foo:2:0:c5', 'gtk:1:0:c3', 'platform:f29:0:c11']), - frozenset(['foo:2:0:c4', 'gtk:2:0:c4', 'platform:f28:0:c10']), - frozenset(['foo:1:0:c2', 'gtk:2:0:c4', 'platform:f28:0:c10']), - frozenset(['foo:2:0:c5', 'gtk:2:0:c5', 'platform:f29:0:c11']), - frozenset(['foo:1:0:c3', 'gtk:2:0:c5', 'platform:f29:0:c11']), - frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10']), - frozenset(['foo:2:0:c4', 'gtk:1:0:c2', 'platform:f28:0:c10']), - frozenset(['foo:1:0:c3', 'gtk:1:0:c3', 'platform:f29:0:c11']) + frozenset(["app:1:0:0:src", + "foo:1:0:c3:x86_64", + "gtk:1:0:c3:x86_64", + "platform:f29:0:c11:x86_64"]), + frozenset(["app:1:0:0:src", + "foo:2:0:c5:x86_64", + "gtk:1:0:c3:x86_64", + "platform:f29:0:c11:x86_64"]), + frozenset(["app:1:0:0:src", + "foo:1:0:c2:x86_64", + "gtk:2:0:c4:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "foo:1:0:c3:x86_64", + "gtk:2:0:c5:x86_64", + "platform:f29:0:c11:x86_64"]), + frozenset(["app:1:0:0:src", + "foo:2:0:c5:x86_64", + "gtk:2:0:c5:x86_64", + "platform:f29:0:c11:x86_64"]), + frozenset(["app:1:0:0:src", + "foo:2:0:c4:x86_64", + "gtk:2:0:c4:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "foo:1:0:c2:x86_64", + "gtk:1:0:c2:x86_64", + "platform:f28:0:c10:x86_64"]), + frozenset(["app:1:0:0:src", + "foo:2:0:c4:x86_64", + "gtk:1:0:c2:x86_64", + "platform:f28:0:c10:x86_64"]), ]) assert expanded == expected