diff --git a/module_build_service/mmd_resolver.py b/module_build_service/mmd_resolver.py index 4a10fc15..536819a9 100644 --- a/module_build_service/mmd_resolver.py +++ b/module_build_service/mmd_resolver.py @@ -27,7 +27,8 @@ import enum import collections import itertools import solv -from module_build_service import log +from module_build_service import log, conf +from module_build_service.models import ModuleBuild class MMDResolverPolicy(enum.Enum): @@ -45,8 +46,11 @@ class MMDResolver(object): self.pool.setarch("x86_64") self.build_repo = self.pool.add_repo("build") self.available_repo = self.pool.add_repo("available") + # Solvable objects representing modules stored in a list grouped by + # the name:stream. + self.solvables = {} - def _deps2reqs(self, deps): + def _deps2reqs(self, deps, base_module_stream_overrides=None, exact_versions=True): """ Helper method converting dependencies from MMD to solv.Dep instance expressing the dependencies in a way libsolv accepts as input. @@ -56,6 +60,24 @@ class MMDResolver(object): The resulting solv.Dep expression will be: ((module(gtk) with module(gtk:1)) and (module(foo) with module(foo:1))) + Base modules are handled in a special way in case when the stream of base module + contains version in the "x.y.z" format. For example "el8.0.0" or "el7.6.0". + In this case, the resulting solv.Dep expression for such base module will contain verison + string computed using ModuleBuild.get_stream_version() method: + For example: + module(platform) with module(platform:el8) = 080200 + + The stream used to compute the version can be also overriden using the + `base_module_stream_overrides` dict which has base module name as a key and + the stream which will be used to compute the version as a value. + This is needed for cases when module requires just "platform:el8", but was + in fact built against particular platform stream, for example platform:el8.1.0. + In this case, such module should still require platform:el8, but in particular + version which is passed to this method using the `base_module_stream_overrides`. + + When `exact_versions` is set to False, the base module dependency will contain + ">=" operator instead of "=". + The "with" syntax is here to allow depending on "module(gtk)" meaning "any gtk". This can happen in case {'gtk': []} is used as an input. @@ -66,6 +88,11 @@ class MMDResolver(object): ``Modulemd.Dependencies.get_requires`` or ``Modulemd.Dependencies.get_buildrequires`` whose value is converted from ``Modulemd.SimpleSet`` to list. + :param dict base_module_stream_overrides: The key is base module name, value + is the stream string which will be used to compute `version` part of the + base module solv.Dep expression. + :param bool exact_versions: When set to False, the base module dependency + will contain ">=" operator instead of "=". :rtype: solv.Dep :return: solv.Dep instance with dependencies in form libsolv accepts. """ @@ -78,6 +105,8 @@ class MMDResolver(object): # or "Requires:". # This method creates such solve.Dep. stream_dep = lambda n, s: pool.Dep("module(%s:%s)" % (n, s)) + versioned_stream_dep = lambda n, s, v, op: pool.Dep("module(%s:%s)" % (n, s)).Rel( + op, pool.Dep(str(v))) # There are relations between modules in `deps`. For example: # deps = [{'gtk': ['1'], 'foo': ['1']}]" means "gtk:1 and foo:1" are both required. @@ -95,6 +124,8 @@ class MMDResolver(object): # Contains the solv.Dep requirements for current dict. require = None for name, streams in dep_dicts.items(): + is_base_module = name in conf.base_module_names + # The req_pos will store solv.Dep expression for "positive" requirements. # That is the case of 'gtk': ['1', '2']. # The req_neg will store negative requirements like 'gtk': ['-1', '-2']. @@ -103,10 +134,65 @@ class MMDResolver(object): # For each stream in `streams` for this dependency, generate the # module(name:stream) solv.Dep and add REL_OR relations between them. for stream in streams: - if stream.startswith("-"): - req_neg = rel_or_dep(req_neg, solv.REL_OR, stream_dep(name, stream[1:])) + if is_base_module: + # Override the stream which is used to compute the stream version in case + # `base_module_stream_overrides` is set. + if base_module_stream_overrides and name in base_module_stream_overrides: + stream_for_version = base_module_stream_overrides[name] + else: + stream_for_version = stream + + # In case x.y.z versioning is not used for this base module, do not + # use versions solv.Dep. + if len(str(ModuleBuild.get_stream_version( + stream_for_version, right_pad=False))) < 5: + if stream.startswith("-"): + req_neg = rel_or_dep( + req_neg, solv.REL_OR, stream_dep(name, stream[1:])) + else: + req_pos = rel_or_dep( + req_pos, solv.REL_OR, stream_dep(name, stream)) + else: + # The main reason why to use `exact_versions` is the case when + # adding deps for the input module we want to resolve. This module + # buildrequires exact stream version of base module against which it + # needs for building and we should never pull in different one. + # But for modules which are buildrequires of this input module, we + # want to use "base_module >= stream_version" relation, because they + # can be chery-picked even when newer base module stream_version is + # requested, for example: + # - foo buildrequires bar and also buildrequires platform:el8 = 080100. + # - bar:1 is built against platform:el8.0.0. + # - bar:2 is built against platform:el8.2.0. + # We need libsolv to allow chery-picking "bar:1", and ignore "bar:2", + # because it is built against newer platform stream version than the + # requested and and such newer version can be incompatible with the + # old one. so we express bar's dependencies on platform like this: + # - bar:1 buildrequires platform:el8 >= 080000. + # - bar:2 buildrequires platform:el8 >= 080200. + # Because the "foo" limits the solving to platform:el8 = 080100, + # the bar:2 won't be returned by libsolv, because 080100 < 080200. + # But that bar:1 will be returned by libsovl, because it buildrequires + # platform 080000 which is lesser than 080100. + op = solv.REL_EQ + if not exact_versions: + op |= solv.REL_GT + version = ModuleBuild.get_stream_version( + stream_for_version, right_pad=False) + if stream.startswith("-"): + req_neg = rel_or_dep( + req_neg, solv.REL_OR, + versioned_stream_dep(name, stream[1:], version, op)) + else: + req_pos = rel_or_dep( + req_pos, solv.REL_OR, + versioned_stream_dep(name, stream, version, op)) else: - req_pos = rel_or_dep(req_pos, solv.REL_OR, stream_dep(name, stream)) + if stream.startswith("-"): + req_neg = rel_or_dep( + req_neg, solv.REL_OR, stream_dep(name, stream[1:])) + else: + req_pos = rel_or_dep(req_pos, solv.REL_OR, stream_dep(name, stream)) # Generate the module(name) solv.Dep. req = pool.Dep("module(%s)" % name) @@ -126,6 +212,64 @@ class MMDResolver(object): return reqs + def _add_virtual_provides(self, solvable, mmd): + """ + Adds "virtual_streams" from XMD section of `mmd` to `solvable`. + + Base modules like "platform" can contain virtual streams which need to be considered + when resolving dependencies. For example module "platform:el8.1.0" can provide virtual + stream "el8". In this case the solvable will have following additional Provides: + + - module(platform:el8.1.0) = 80100 - Modules can require specific platform stream. + - module(platform:el8) = 80100 - Module can also require just platform:el8. + """ + xmd = mmd.get_xmd() + # Return in case virtual_streams are not set for this mmd. + if "mbs" not in xmd.keys() or "virtual_streams" not in xmd["mbs"].keys(): + return + + # When depsolving, we will need to follow specific rules to choose the right base + # module, like sorting the base modules sharing the same virtual streams based on + # their "stream version" - For example stream "el8.1" is lower than stream "el8.2" + # and so on. We therefore need to convert the stream and version of base module to + # integer representation and add "module($name:$stream) = $stream_based_version" + # to Provides. + version = ModuleBuild.get_stream_version( + mmd.get_stream(), right_pad=False) + dep = self.pool.Dep("module(%s:%s)" % (mmd.get_name(), mmd.get_stream())).Rel( + solv.REL_EQ, self.pool.Dep(str(version))) + solvable.add_deparray(solv.SOLVABLE_PROVIDES, dep) + + # For each virtual stream, add + # "module($name:$stream) = $virtual_stream_based_version" provide. + for stream in xmd["mbs"]["virtual_streams"]: + dep = self.pool.Dep("module(%s:%s)" % (mmd.get_name(), stream)).Rel( + solv.REL_EQ, self.pool.Dep(str(version))) + solvable.add_deparray(solv.SOLVABLE_PROVIDES, dep) + + def _get_base_module_stream_overrides(self, mmd): + """ + Checks the xmd["mbs"]["buildrequires"] and returns the dict containing + base module name as a key and stream of base module against which this + module was built. This is later used to override base module streams + in the _deps2reqs method. + + :param Modulemd mmd: Metadata of module for which the stream overrides are returned. + :rtype: dict + :return: Dict with module name as a key and new stream as a value. + """ + overrides = {} + if "mbs" in mmd.get_xmd().keys(): + for base_module_name in conf.base_module_names: + if base_module_name not in mmd.get_xmd()["mbs"]["buildrequires"].keys(): + continue + if "stream" not in mmd.get_xmd()["mbs"]["buildrequires"][base_module_name].keys(): + continue + stream = mmd.get_xmd()["mbs"]["buildrequires"][base_module_name]["stream"] + + overrides[base_module_name] = stream + return overrides + def add_modules(self, mmd): """ Adds module represented by `mmd` metadata to MMDResolver. Modules added by this @@ -147,6 +291,8 @@ class MMDResolver(object): for name, streams in getattr(dep, fn)().items()} for dep in mmd.get_dependencies()] + base_module_stream_overrides = self._get_base_module_stream_overrides(mmd) + # Each solvable object has name, version, architecture and list of # provides/requires/conflicts which defines its relations with other solvables. # You can imagine solvable as a single RPM. @@ -177,14 +323,24 @@ class MMDResolver(object): pool.Dep("module(%s:%s)" % (n, s)).Rel( solv.REL_EQ, pool.Dep(str(v)))) + self._add_virtual_provides(solvable, mmd) + # Fill in the "Requires" of this module, so we can track its dependencies # on other modules. - requires = self._deps2reqs(normdeps(mmd, "get_requires")) + requires = self._deps2reqs(normdeps(mmd, "get_requires"), + base_module_stream_overrides, False) + log.debug("Adding module %s with requires: %r", solvable.name, requires) solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires) # Add "Conflicts: module(name)", because TODO, ask ignatenko. solvable.add_deparray(solv.SOLVABLE_CONFLICTS, pool.Dep("module(%s)" % n)) solvables.append(solvable) + + # Add solvable to solvables list. Sorting is done later in the solve method. + ns = ":".join([n, s]) + if ns not in self.solvables: + self.solvables[ns] = [] + self.solvables[ns].append(solvable) else: # For input module, we might have multiple buildrequires/requires pairs in the # input `mmd`. For example like this: @@ -224,10 +380,16 @@ class MMDResolver(object): solvable.evr = str(v) solvable.arch = "src" - requires = self._deps2reqs([normalized_deps[c]]) + requires = self._deps2reqs([normalized_deps[c]], base_module_stream_overrides) + log.debug("Adding module %s with requires: %r", solvable.name, requires) solvable.add_deparray(solv.SOLVABLE_REQUIRES, requires) solvables.append(solvable) + # Add solvable to solvables list. Sorting is done later in the solve method. + ns = ":".join([n, s]) + if ns not in self.solvables: + self.solvables[ns] = [] + self.solvables[ns].append(solvable) return solvables @@ -267,6 +429,10 @@ class MMDResolver(object): # "solvable to n:s" s2ns = lambda s: ":".join(s.name.split(":", 2)[:2]) + # Sort the solvables by version for NSVC in descending order. + for ns, unordered_solvables in self.solvables.items(): + unordered_solvables.sort(key=lambda s: int(s.name.split(":")[2]), reverse=True) + # For each solvable object generated from input module, run the solver. # For reasons why there might be multiple solvable objects, please read the # `add_modules(...)` inline comments. @@ -359,12 +525,27 @@ class MMDResolver(object): # of our jobs - those are the modules we are looking for. newsolvables = solver.transaction().newsolvables() log.debug("Transaction:") + for s in newsolvables: log.debug(" - %s", s) + + # Skip this alternative in case not all the favored Solvables are in + # the solution. For example if we favored gtk:4:1:c8, but it simply + # cannot be installed because of other dependencies, we know this is not a + # possible combination and we should not treat it as an alternative. + all_solvables_found = True + for s in opt: + if s not in newsolvables: + all_solvables_found = False + break + # Append them as an alternative for this src_alternative. # Remember that src_alternatives are grouped by NS or NSVC depending on # MMDResolverPolicy, so there might be more of them. - alternative.append(newsolvables) + if all_solvables_found: + alternative.append(newsolvables) + else: + log.debug(" - ^ Not all favored solvables found in the result, skipping.") # If the MMDResolverPolicy is First, we will check all the alternatives and keep # just the "first" one. @@ -372,15 +553,37 @@ class MMDResolver(object): # Prune for transactions in alternatives.values(): for ns, trans in transactions.items(): - try: - # The transation to keep is defined by the name:stream comparison, - # so we always return the same name:stream if the input is the same. - transactions[ns] = [next(t for t in trans - if set(ns) <= set(s2ns(s) for s in t))] - except StopIteration: - # No transactions found for requested N:S - del transactions[ns] - continue + # Each transaction in trans lists all the possible working + # combination of solvables. Our goal here is to find out the + # transaction which installs the most latest Solvables - ideally + # always the latest versions of the Solvables we have, but this might + # not be always possible because of dependencies. + # + # We achieve that by generating sorted_trans list in follwing format: + # [[transaction_id, [solvable1_index, solvable2_index, ...]], [...], ...] + # + # The solvableN_index is a number saying how new the solvable is. We use + # `self.solvables` to get that number and it is simply index + # of the solvable in the particular self.solvables[name_stream] list. + # The newest solvable has therefore index 0, the next newest solvable index 1 + # and so on. + # + # Then we simply sort the `sorted_trans` based on the sum of solvableN_index + # which gives us the transaction with the most recent versions. This is + # used as a solution. + sorted_trans = [] + for i, t in enumerate(trans): + idx = [] + for s in t: + name_stream = s2ns(s) + if name_stream not in self.solvables: + continue + index = self.solvables[name_stream].index(s) + idx.append(index) + sorted_trans.append([i, idx]) + sorted_trans.sort(key=lambda i: sum(i[1])) + if sorted_trans: + transactions[ns] = [trans[sorted_trans[0][0]]] # Convert the solvables in alternatives to nsvc and return them as set of frozensets. return set(frozenset(s2nsvc(s) for s in transactions[0]) diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index f97ba689..3b877ffc 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -222,11 +222,13 @@ def get_prefixed_version(mmd): """ xmd = mmd.get_xmd() version = mmd.get_version() + base_module_stream = None for base_module in conf.base_module_names: # xmd is a GLib Variant and doesn't support .get() syntax try: - base_module_stream = xmd['mbs']['buildrequires'].get(base_module, {}).get('stream') + base_module_stream = xmd['mbs']['buildrequires'].get( + base_module, {}).get('stream') if base_module_stream: # Break after finding the first base module that is buildrequired break diff --git a/tests/test_mmd_resolver.py b/tests/test_mmd_resolver.py index bfe0ff53..58902e7c 100644 --- a/tests/test_mmd_resolver.py +++ b/tests/test_mmd_resolver.py @@ -28,6 +28,7 @@ import pytest from module_build_service.mmd_resolver import MMDResolver from module_build_service import Modulemd +from module_build_service import glib class TestMMDResolver: @@ -39,7 +40,7 @@ class TestMMDResolver: pass @staticmethod - def _make_mmd(nsvc, requires): + def _make_mmd(nsvc, requires, xmd_buildrequires=None, virtual_streams=None): name, stream, version = nsvc.split(":", 2) mmd = Modulemd.Module() mmd.set_mdversion(2) @@ -51,6 +52,17 @@ class TestMMDResolver: licenses.add("GPL") mmd.set_module_licenses(licenses) + xmd = glib.from_variant_dict(mmd.get_xmd()) + xmd["mbs"] = {} + xmd["mbs"]["buildrequires"] = {} + if xmd_buildrequires: + for ns in xmd_buildrequires: + n, s = ns.split(":") + xmd["mbs"]["buildrequires"][n] = {"stream": s} + if virtual_streams: + xmd["mbs"]["virtual_streams"] = virtual_streams + mmd.set_xmd(glib.dict_values(xmd)) + if ":" in version: version, context = version.split(":") mmd.set_context(context) @@ -162,3 +174,89 @@ class TestMMDResolver: for e in exp) assert expanded == expected + + @pytest.mark.parametrize( + "buildrequires, xmd_buildrequires, expected", ( + # BR all platform streams -> build for all platform streams. + ({"platform": []}, {}, [ + [["platform:el8.2.0:0:c0:x86_64"], + ["platform:el8.1.0:0:c0:x86_64"], + ["platform:el8.0.0:0:c0:x86_64"], + ["platform:el7.6.0:0:c0:x86_64"]], + ]), + # BR "el8" platform stream -> build for all el8 platform streams. + ({"platform": ["el8"]}, {}, [ + [["platform:el8.2.0:0:c0:x86_64"], + ["platform:el8.1.0:0:c0:x86_64"], + ["platform:el8.0.0:0:c0:x86_64"]], + ]), + # BR "el8.1.0" platfrom stream -> build just for el8.1.0. + ({"platform": ["el8"]}, ["platform:el8.1.0"], [ + [["platform:el8.1.0:0:c0:x86_64"]], + ]), + # BR platform:el8.1.0 and gtk:3, which is not built against el8.1.0, + # but it is built only against el8.0.0 -> cherry-pick gtk:3 from el8.0.0 + # and build once against platform:el8.1.0. + ({"platform": ["el8"], "gtk": ["3"]}, ["platform:el8.1.0"], [ + [["platform:el8.1.0:0:c0:x86_64", "gtk:3:0:c8:x86_64", ]], + ]), + # BR platform:el8.2.0 and gtk:3, this time gtk:3 build against el8.2.0 exists + # -> use both platform and gtk from el8.2.0 and build once. + ({"platform": ["el8"], "gtk": ["3"]}, ["platform:el8.2.0"], [ + [["platform:el8.2.0:0:c0:x86_64", "gtk:3:1:c8:x86_64", ]], + ]), + # BR platform:el8.2.0 and mess:1 which is built against platform:el8.1.0 and + # requires gtk:3 which is built against platform:el8.2.0 and platform:el8.0.0 + # -> Use platform:el8.2.0 and + # -> cherry-pick mess:1 from el8.1.0 and + # -> use gtk:3:1 from el8.2.0. + ({"platform": ["el8"], "mess": ["1"]}, ["platform:el8.2.0"], [ + [["platform:el8.2.0:0:c0:x86_64", "mess:1:0:c0:x86_64", "gtk:3:1:c8:x86_64", ]], + ]), + # BR platform:el8.1.0 and mess:1 which is built against platform:el8.1.0 and + # requires gtk:3 which is built against platform:el8.2.0 and platform:el8.0.0 + # -> Use platform:el8.1.0 and + # -> Used mess:1 from el8.1.0 and + # -> cherry-pick gtk:3:0 from el8.0.0. + ({"platform": ["el8"], "mess": ["1"]}, ["platform:el8.1.0"], [ + [["platform:el8.1.0:0:c0:x86_64", "mess:1:0:c0:x86_64", "gtk:3:0:c8:x86_64", ]], + ]), + # BR platform:el8.0.0 and mess:1 which is built against platform:el8.1.0 and + # requires gtk:3 which is built against platform:el8.2.0 and platform:el8.0.0 + # -> No valid combination, because mess:1 is only available in el8.1.0 and later. + ({"platform": ["el8"], "mess": ["1"]}, ["platform:el8.0.0"], []), + # This is undefined... it might build just once against latest platform or + # against all the platforms... we don't know + # ({"platform": ["el8"], "gtk": ["3"]}, {}, [ + # [["platform:el8.2.0:0:c0:x86_64", "gtk:3:1:c8:x86_64"]], + # ]), + ) + ) + def test_solve_virtual_streams(self, buildrequires, xmd_buildrequires, expected): + modules = ( + # (nsvc, buildrequires, expanded_buildrequires, virtual_streams) + ("platform:el8.0.0:0:c0", {}, {}, ["el8"]), + ("platform:el8.1.0:0:c0", {}, {}, ["el8"]), + ("platform:el8.2.0:0:c0", {}, {}, ["el8"]), + ("platform:el7.6.0:0:c0", {}, {}, ["el7"]), + ("gtk:3:0:c8", {"platform": ["el8"]}, {"platform:el8.0.0"}, None), + ("gtk:3:1:c8", {"platform": ["el8"]}, {"platform:el8.2.0"}, None), + ("mess:1:0:c0", [{"gtk": ["3"], "platform": ["el8"]}], {"platform:el8.1.0"}, None), + ) + for n, req, xmd_br, virtual_streams in modules: + self.mmd_resolver.add_modules(self._make_mmd( + n, req, xmd_br, virtual_streams)) + + app = self._make_mmd("app:1:0", buildrequires, xmd_buildrequires) + if not expected: + with pytest.raises(RuntimeError): + self.mmd_resolver.solve(app) + return + else: + expanded = self.mmd_resolver.solve(app) + + expected = set(frozenset(["app:1:0:%d:src" % c] + e) + for c, exp in enumerate(expected) + for e in exp) + + assert expanded == expected diff --git a/tests/test_utils/test_utils_mse.py b/tests/test_utils/test_utils_mse.py index 7c8f74a6..a0a22fd8 100644 --- a/tests/test_utils/test_utils_mse.py +++ b/tests/test_utils/test_utils_mse.py @@ -70,8 +70,8 @@ class TestUtilsModuleStreamExpansion: xmd = { "mbs": { - "buildrequires": [], - "requires": [], + "buildrequires": {}, + "requires": {}, "commit": "ref_%s" % context, "mse": "true", }