mmd_resolver: rework solvables generation

Now we have good separation between runtime and buildtime dependencies:
If mmd has context, it is built artefact which supposed to have only one
dependency entry, we create solvable with real arch for it. Otherwise we
create multiple solvables with "src" arch.

In "src" solvables, "context" is a number which represents index in
dependencies array.

Stream is not included anymore in "evr" because it's incomparable.
Version is not included anymore in "name" because it's there just for
comparison, nothing more (it's not different module-stream).

As a side, add_requires/add_conflicts/add_provides were replaced with
non-deprecated add_deparray method.

With all this, we are ready to handle MMDs which have multiple elements
in dependencies (modulemd v2).

Signed-off-by: Igor Gnatenko <ignatenko@redhat.com>
This commit is contained in:
Igor Gnatenko
2018-02-28 19:42:48 +01:00
committed by mprahl
parent 060275f094
commit 7cbd0a0301
2 changed files with 203 additions and 119 deletions

View File

@@ -115,68 +115,75 @@ class MMDResolver(object):
self.build_repo = self.pool.add_repo("build")
self.available_repo = self.pool.add_repo("available")
self.alternatives_whitelist = set()
def add_modules(self, mmd):
n, s, v, c = mmd.get_name(), mmd.get_stream(), mmd.get_version(), mmd.get_context()
def _create_solvable(self, repo, mmd):
"""
Creates libsolv Solvable object in repo `repo` based on the Modulemd
metadata `mmd`.
pool = self.pool
This fills in all the provides/requires/conflicts of Solvable.
solvables = []
if c is not None:
# Built module
deps = mmd.get_dependencies()
if len(deps) > 1:
assert False, "Multiple dependencies for runtime"
:rtype: solv.Solvable
:return: Solvable object.
"""
solvable = repo.add_solvable()
solvable.name = "%s:%s:%d:%s" % (mmd.get_name(), mmd.get_stream(),
mmd.get_version(), mmd.get_context())
solvable.evr = "%s-%d" % (mmd.get_stream(), mmd.get_version())
solvable.arch = "x86_64"
# $n:$s:$c-$v.$a
solvable = self.available_repo.add_solvable()
solvable.name = "%s:%s:%d:%s" % (n, s, v, c)
solvable.evr = str(v)
# TODO: replace with real arch
solvable.arch = "x86_64"
# Provides
solvable.add_provides(
self.pool.Dep("module(%s)" % mmd.get_name()).Rel(
solv.REL_EQ, self.pool.Dep(solvable.evr)))
# Prv: module($n)
solvable.add_deparray(solv.SOLVABLE_PROVIDES,
pool.Dep("module(%s)" % n))
# Prv: module($n:$s) = $v
solvable.add_deparray(solv.SOLVABLE_PROVIDES,
pool.Dep("module(%s:%s)" % (n, s)).Rel(
solv.REL_EQ, pool.Dep(str(v))))
# Requires
for deps in mmd.get_dependencies():
for name, streams in deps.get_requires().items():
requires = None
for stream in streams.get():
require = self.pool.Dep("module(%s)" % name)
require = require.Rel(solv.REL_EQ, self.pool.Dep(stream))
if requires:
requires = requires.Rel(solv.REL_OR, require)
else:
requires = require
solvable.add_requires(requires)
if deps:
# Req: (module($on1:$os1) OR module($on2:$os2) OR …)
for name, streams in deps[0].get_requires().items():
requires = None
for stream in streams.get():
require = pool.Dep("module(%s:%s)" % (name, stream))
if requires is not None:
requires = requires.Rel(solv.REL_OR, require)
else:
requires = require
solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires)
# Build-requires in case we are in build_repo.
if repo == self.build_repo:
solvable.arch = "src"
for deps in mmd.get_dependencies():
# Con: module($n)
solvable.add_deparray(solv.SOLVABLE_CONFLICTS, pool.Dep("module(%s)" % n))
solvables.append(solvable)
else:
# Input module
# Context means two things:
# * Unique identifier
# * Offset for the dependency which was used
for c, deps in enumerate(mmd.get_dependencies()):
# $n:$s:$c-$v.src
solvable = self.build_repo.add_solvable()
solvable.name = "%s:%s:%s:%d" % (n, s, v, c)
solvable.evr = str(v)
solvable.arch = "src"
# Req: (module($on1:$os1) OR module($on2:$os2) OR …)
for name, streams in deps.get_buildrequires().items():
requires = None
for stream in streams.get():
require = self.pool.Dep("module(%s)" % name)
require = require.Rel(solv.REL_EQ, self.pool.Dep(stream))
require = pool.Dep("module(%s:%s)" % (name, stream))
if requires:
requires = requires.Rel(solv.REL_OR, require)
else:
requires = require
solvable.add_requires(requires)
self.alternatives_whitelist.add(name)
solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires)
# Conflicts
solvable.add_conflicts(self.pool.Dep("module(%s)" % mmd.get_name()))
solvables.append(solvable)
return solvable
def add_available_module(self, mmd):
"""
Adds module available for dependency solving.
"""
self._create_solvable(self.available_repo, mmd)
return solvables
def solve(self, mmd):
"""
@@ -187,13 +194,22 @@ class MMDResolver(object):
:return: set of frozensets of n:s:v:c of modules which satisfied the
dependency solving.
"""
solvable = self._create_solvable(self.build_repo, mmd)
solvables = self.add_modules(mmd)
if not solvables:
raise ValueError("No module(s) found for resolving")
self.pool.createwhatprovides()
new_job = self.pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE, solvable.id)
# XXX: Using SOLVABLE_ONE_OF should be faster & more convenient.
# There must be a bug in _gather_alternatives(), possibly when processing l1 alternatives.
# Use pool.towhatprovides() to combine solvables.
alternatives = []
jobs = self.pool.getpooljobs()
self.pool.setpooljobs(jobs + [new_job])
alternatives = _gather_alternatives(self.pool)
for s in solvables:
new_job = self.pool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE, s.id)
self.pool.setpooljobs(jobs + [new_job])
alternatives.extend(_gather_alternatives(self.pool))
self.pool.setpooljobs(jobs)
return set(frozenset(s.name for s in trans if s != solvable) for trans in alternatives)
return set(frozenset("%s:%s" % (s.name, s.arch) for s in trans)
for trans in alternatives)

View File

@@ -24,7 +24,7 @@
# Igor Gnatenko <ignatenko@redhat.com>
import gi
gi.require_version('Modulemd', '1.0') # noqa
gi.require_version("Modulemd", "1.0") # noqa
from gi.repository import Modulemd
from module_build_service.mmd_resolver import MMDResolver
@@ -38,14 +38,12 @@ class TestMMDResolver:
def teardown_method(self, test_method):
pass
def _make_mmd(self, nsvc, requires, build_requires):
name, stream, version, context = nsvc.split(":")
def _make_mmd(self, nsvc, requires):
name, stream, version = nsvc.split(":", 2)
mmd = Modulemd.Module()
mmd.set_mdversion(2)
mmd.set_name(name)
mmd.set_stream(stream)
mmd.set_version(int(version))
mmd.set_context(context)
mmd.set_summary("foo")
mmd.set_description("foo")
licenses = Modulemd.SimpleSet()
@@ -53,98 +51,168 @@ class TestMMDResolver:
mmd.set_module_licenses(licenses)
deps = Modulemd.Dependencies()
if ":" in version:
version, context = version.split(":")
mmd.set_context(context)
add_requires = deps.add_requires
else:
add_requires = deps.add_buildrequires
mmd.set_version(int(version))
for req_name, req_streams in requires.items():
deps.add_requires(req_name, req_streams)
for req_name, req_streams in build_requires.items():
deps.add_buildrequires(req_name, req_streams)
add_requires(req_name, req_streams)
mmd.set_dependencies((deps, ))
return mmd
def test_solve_tree(self):
mmds = []
mmds.append(self._make_mmd("app:1:0:c1", {}, {"gtk": ["1", "2"]}))
mmds.append(self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}, {}))
mmds.append(self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}, {}))
mmds.append(self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}, {}))
mmds.append(self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}, {}))
mmds.append(self._make_mmd("font:a:0:c6", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("font:a:0:c7", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("font:b:0:c8", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("font:b:0:c9", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("platform:f28:0:c10", {}, {}))
mmds.append(self._make_mmd("platform:f29:0:c11", {}, {}))
mmds = [
self._make_mmd("app:1:0", {"gtk": ["1", "2"]}),
self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}),
self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}),
self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}),
self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}),
self._make_mmd("font:a:0:c6", {"platform": ["f28"]}),
self._make_mmd("font:a:0:c7", {"platform": ["f29"]}),
self._make_mmd("font:b:0:c8", {"platform": ["f28"]}),
self._make_mmd("font:b:0:c9", {"platform": ["f29"]}),
self._make_mmd("platform:f28:0:c10", {}),
self._make_mmd("platform:f29:0:c11", {}),
]
for mmd in mmds[1:]:
self.mmd_resolver.add_available_module(mmd)
self.mmd_resolver.add_modules(mmd)
expanded = self.mmd_resolver.solve(mmds[0])
expected = set([
frozenset(["gtk:1:0:c2", "font:a:0:c6", "platform:f28:0:c10"]),
frozenset(["gtk:1:0:c2", "font:b:0:c8", "platform:f28:0:c10"]),
frozenset(["gtk:1:0:c3", "font:a:0:c7", "platform:f29:0:c11"]),
frozenset(["gtk:1:0:c3", "font:b:0:c9", "platform:f29:0:c11"]),
frozenset(["gtk:2:0:c4", "font:a:0:c6", "platform:f28:0:c10"]),
frozenset(["gtk:2:0:c4", "font:b:0:c8", "platform:f28:0:c10"]),
frozenset(["gtk:2:0:c5", "font:a:0:c7", "platform:f29:0:c11"]),
frozenset(["gtk:2:0:c5", "font:b:0:c9", "platform:f29:0:c11"]),
frozenset(["app:1:0:0:src",
"font:a:0:c6:x86_64",
"gtk:1:0:c2:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"font:a:0:c7:x86_64",
"gtk:1:0:c3:x86_64",
"platform:f29:0:c11:x86_64"]),
frozenset(["app:1:0:0:src",
"font:b:0:c8:x86_64",
"gtk:1:0:c2:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"font:b:0:c9:x86_64",
"gtk:1:0:c3:x86_64",
"platform:f29:0:c11:x86_64"]),
frozenset(["app:1:0:0:src",
"font:a:0:c6:x86_64",
"gtk:2:0:c4:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"font:b:0:c9:x86_64",
"gtk:2:0:c5:x86_64",
"platform:f29:0:c11:x86_64"]),
frozenset(["app:1:0:0:src",
"font:b:0:c8:x86_64",
"gtk:2:0:c4:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"font:a:0:c7:x86_64",
"gtk:2:0:c5:x86_64",
"platform:f29:0:c11:x86_64"]),
])
assert expanded == expected
def test_solve_tree_buildrequire_platform(self):
mmds = []
mmds.append(self._make_mmd("app:1:0:c1", {}, {"gtk": ["1", "2"], "platform": ["f28"]}))
mmds.append(self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}, {}))
mmds.append(self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}, {}))
mmds.append(self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}, {}))
mmds.append(self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}, {}))
mmds.append(self._make_mmd("font:a:0:c6", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("font:a:0:c7", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("font:b:0:c8", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("font:b:0:c9", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("platform:f28:0:c10", {}, {}))
mmds.append(self._make_mmd("platform:f29:0:c11", {}, {}))
mmds = [
self._make_mmd("app:1:0", {"gtk": ["1", "2"], "platform": ["f28"]}),
self._make_mmd("gtk:1:0:c2", {"font": ["a", "b"], "platform": ["f28"]}),
self._make_mmd("gtk:1:0:c3", {"font": ["a", "b"], "platform": ["f29"]}),
self._make_mmd("gtk:2:0:c4", {"font": ["a", "b"], "platform": ["f28"]}),
self._make_mmd("gtk:2:0:c5", {"font": ["a", "b"], "platform": ["f29"]}),
self._make_mmd("font:a:0:c6", {"platform": ["f28"]}),
self._make_mmd("font:a:0:c7", {"platform": ["f29"]}),
self._make_mmd("font:b:0:c8", {"platform": ["f28"]}),
self._make_mmd("font:b:0:c9", {"platform": ["f29"]}),
self._make_mmd("platform:f28:0:c10", {}),
self._make_mmd("platform:f29:0:c11", {}),
]
for mmd in mmds[1:]:
self.mmd_resolver.add_available_module(mmd)
self.mmd_resolver.add_modules(mmd)
expanded = self.mmd_resolver.solve(mmds[0])
expected = set([
frozenset(["gtk:1:0:c2", "platform:f28:0:c10", "font:a:0:c6"]),
frozenset(["gtk:1:0:c2", "platform:f28:0:c10", "font:b:0:c8"]),
frozenset(["gtk:2:0:c4", "platform:f28:0:c10", "font:a:0:c6"]),
frozenset(["gtk:2:0:c4", "platform:f28:0:c10", "font:b:0:c8"]),
frozenset(["app:1:0:0:src",
"font:a:0:c6:x86_64",
"gtk:2:0:c4:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"font:b:0:c8:x86_64",
"gtk:1:0:c2:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"font:a:0:c6:x86_64",
"gtk:1:0:c2:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"font:b:0:c8:x86_64",
"gtk:2:0:c4:x86_64",
"platform:f28:0:c10:x86_64"]),
])
assert expanded == expected
def test_solve_tree_multiple_build_requires(self):
mmds = []
mmds.append(self._make_mmd("app:1:0:c1", {}, {"gtk": ["1", "2"], "foo": ["1", "2"]}))
mmds.append(self._make_mmd("gtk:1:0:c2", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("gtk:1:0:c3", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("gtk:2:0:c4", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("gtk:2:0:c5", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("foo:1:0:c2", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("foo:1:0:c3", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("foo:2:0:c4", {"platform": ["f28"]}, {}))
mmds.append(self._make_mmd("foo:2:0:c5", {"platform": ["f29"]}, {}))
mmds.append(self._make_mmd("platform:f28:0:c10", {}, {}))
mmds.append(self._make_mmd("platform:f29:0:c11", {}, {}))
mmds = [
self._make_mmd("app:1:0", {"gtk": ["1", "2"], "foo": ["1", "2"]}),
self._make_mmd("gtk:1:0:c2", {"platform": ["f28"]}),
self._make_mmd("gtk:1:0:c3", {"platform": ["f29"]}),
self._make_mmd("gtk:2:0:c4", {"platform": ["f28"]}),
self._make_mmd("gtk:2:0:c5", {"platform": ["f29"]}),
self._make_mmd("foo:1:0:c2", {"platform": ["f28"]}),
self._make_mmd("foo:1:0:c3", {"platform": ["f29"]}),
self._make_mmd("foo:2:0:c4", {"platform": ["f28"]}),
self._make_mmd("foo:2:0:c5", {"platform": ["f29"]}),
self._make_mmd("platform:f28:0:c10", {}),
self._make_mmd("platform:f29:0:c11", {}),
]
for mmd in mmds[1:]:
self.mmd_resolver.add_available_module(mmd)
self.mmd_resolver.add_modules(mmd)
expanded = self.mmd_resolver.solve(mmds[0])
expected = set([
frozenset(['foo:2:0:c5', 'gtk:1:0:c3', 'platform:f29:0:c11']),
frozenset(['foo:2:0:c4', 'gtk:2:0:c4', 'platform:f28:0:c10']),
frozenset(['foo:1:0:c2', 'gtk:2:0:c4', 'platform:f28:0:c10']),
frozenset(['foo:2:0:c5', 'gtk:2:0:c5', 'platform:f29:0:c11']),
frozenset(['foo:1:0:c3', 'gtk:2:0:c5', 'platform:f29:0:c11']),
frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10']),
frozenset(['foo:2:0:c4', 'gtk:1:0:c2', 'platform:f28:0:c10']),
frozenset(['foo:1:0:c3', 'gtk:1:0:c3', 'platform:f29:0:c11'])
frozenset(["app:1:0:0:src",
"foo:1:0:c3:x86_64",
"gtk:1:0:c3:x86_64",
"platform:f29:0:c11:x86_64"]),
frozenset(["app:1:0:0:src",
"foo:2:0:c5:x86_64",
"gtk:1:0:c3:x86_64",
"platform:f29:0:c11:x86_64"]),
frozenset(["app:1:0:0:src",
"foo:1:0:c2:x86_64",
"gtk:2:0:c4:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"foo:1:0:c3:x86_64",
"gtk:2:0:c5:x86_64",
"platform:f29:0:c11:x86_64"]),
frozenset(["app:1:0:0:src",
"foo:2:0:c5:x86_64",
"gtk:2:0:c5:x86_64",
"platform:f29:0:c11:x86_64"]),
frozenset(["app:1:0:0:src",
"foo:2:0:c4:x86_64",
"gtk:2:0:c4:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"foo:1:0:c2:x86_64",
"gtk:1:0:c2:x86_64",
"platform:f28:0:c10:x86_64"]),
frozenset(["app:1:0:0:src",
"foo:2:0:c4:x86_64",
"gtk:1:0:c2:x86_64",
"platform:f28:0:c10:x86_64"]),
])
assert expanded == expected