From e22cbbab31334dfda17ab2574802c86356d8c416 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Thu, 1 Mar 2018 13:18:06 +0100 Subject: [PATCH] Add get_modules_build_required_by_module_recursively to get the input for libsolv solver. --- module_build_service/utils.py | 103 ++++++++++++++ tests/test_utils/test_utils_mse.py | 209 +++++++++++++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 tests/test_utils/test_utils_mse.py diff --git a/module_build_service/utils.py b/module_build_service/utils.py index e03e719c..9e512cf4 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -1404,6 +1404,109 @@ def get_reusable_component(session, module, component_name, return reusable_component +def _get_mmds_from_requires(session, requires, mmds, recursive=False): + """ + Helper method for get_modules_build_required_by_module_recursively returning + the list of module metadata objects defined by `requires` dict. + + :param session: SQLAlchemy DB session. + :param requires: Modulemetadata requires or buildrequires. + :param mmds: Dictionary with already handled name:streams as a keys and lists + of resulting mmds as values. + :param recursive: If True, the requires are checked recursively. + :return: Dict with name:stream as a key and list with mmds as value. + """ + # To be able to call itself recursively, we need to store list of mmds + # we have added to global mmds list in this particular call. + added_mmds = {} + for name, streams in requires: + # Stream can be prefixed with '-' sign to define that this stream should + # not appear in a resulting list of streams. There can be two situations: + # a) all streams have '-' prefix. In this case, we treat list of streams + # as blacklist and we find all the valid streams and just remove those with + # '-' prefix. + # b) there is at least one stream without '-' prefix. In this case, we can + # ignore all the streams with '-' prefix and just add those without + # '-' prefix to the list of valid streams. + streams_is_blacklist = all([stream[0] == "-" for stream in streams.get()]) + if streams_is_blacklist or len(streams.get()) == 0: + builds = models.ModuleBuild.get_last_build_in_all_streams( + session, name) + valid_streams = [build.stream for build in builds] + else: + valid_streams = [] + for stream in streams.get(): + if stream.startswith("-"): + if streams_is_blacklist and stream[1:] in valid_streams: + valid_streams.remove(stream[1:]) + else: + valid_streams.append(stream) + + # For each valid stream, find the last build in a stream and also all + # its contexts and add mmds of these builds to `mmds` and `added_mmds`. + # Of course only do that if we have not done that already in some + # previous call of this method. + for stream in valid_streams: + ns = "%s:%s" % (name, stream) + if ns in mmds: + continue + + last_build_in_stream = models.ModuleBuild.get_last_build_in_stream( + session, name, stream) + builds = models.ModuleBuild.get_builds_in_version( + session, name, stream, last_build_in_stream.version) + mmds[ns] = [build.mmd() for build in builds] + added_mmds[ns] = mmds[ns] + + # Get the requires recursively. + if recursive: + for mmd_list in added_mmds.values(): + for mmd in mmd_list: + for deps in mmd.get_dependencies(): + mmds = _get_mmds_from_requires(session, deps.get_requires().items(), mmds, True) + + return mmds + + +def get_modules_build_required_by_module_recursively(session, mmd): + """ + Returns the list of Module metadata objects of all modules required while + building the module defined by `mmd` module metadata. + + This method finds out latest versions of all the build-requires of + the `mmd` module and then also all contexts of these latest versions. + + For each build-required name:stream:version:context module, it checks + recursively all the "requires" and finds the latest version of each + required module and also all contexts of these latest versions. + + :rtype: list of Modulemd metadata + :return: List of all modulemd metadata of all modules required to build + the module `mmd`. + """ + # We use dict with name:stream as a key and list with mmds as value. + # That way, we can ensure we won't have any duplicate mmds in a resulting + # list and we also don't waste resources on getting the modules we already + # handled from DB. + mmds = {} + + # At first get all the buildrequires of the module of interest. + for deps in mmd.get_dependencies(): + mmds = _get_mmds_from_requires(session, deps.get_buildrequires().items(), mmds) + + # Now get the requires of buildrequires recursively. + for mmd_key in list(mmds.keys()): + for mmd in mmds[mmd_key]: + for deps in mmd.get_dependencies(): + mmds = _get_mmds_from_requires(session, deps.get_requires().items(), mmds, True) + + # Make single list from dict of lists. + res = [] + for mmds_list in mmds.values(): + res += mmds_list + return res + + def validate_koji_tag(tag_arg_names, pre='', post='-', dict_key='name'): """ Used as a decorator validates koji tag arg(s)' value(s) diff --git a/tests/test_utils/test_utils_mse.py b/tests/test_utils/test_utils_mse.py new file mode 100644 index 00000000..5229243a --- /dev/null +++ b/tests/test_utils/test_utils_mse.py @@ -0,0 +1,209 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import gi +gi.require_version('Modulemd', '1.0') # noqa +from gi.repository import Modulemd +import module_build_service.utils +from module_build_service import models, conf +from tests import (db, clean_database) +from datetime import datetime +import hashlib +from mock import patch +import pytest + + +class TestUtilsModuleStreamExpansion: + + def setup_method(self, test_method): + clean_database() + + def mocked_context(modulebuild_instance): + """ + Changes the ModuleBuild.context behaviour to return + ModuleBuild.build_context instead of computing new context hash. + """ + return modulebuild_instance.build_context + + # For these tests, we need the ModuleBuild.context to return the well-known + # context as we define it in test data. Therefore patch the ModuleBuild.context + # to return ModuleBuild.build_context, which we can control. + self.modulebuild_context_patcher = patch( + "module_build_service.models.ModuleBuild.context", autospec=True) + modulebuild_context = self.modulebuild_context_patcher.start() + modulebuild_context.side_effect = mocked_context + + def teardown_method(self, test_method): + clean_database() + self.modulebuild_context_patcher.stop() + + def _make_module(self, nsvc, requires_list, build_requires_list): + """ + Creates new models.ModuleBuild defined by `nsvc` string with requires + and buildrequires set according to `requires_list` and `build_requires_list`. + + :param str nsvc: name:stream:version:context of a module. + :param list_of_dicts requires_list: List of dictionaries defining the + requires in the mmd requires field format. + :param list_of_dicts build_requires_list: List of dictionaries defining the + build_requires_list in the mmd build_requires_list field format. + :rtype: ModuleBuild + :return: New Module Build. + """ + name, stream, version, context = nsvc.split(":") + mmd = Modulemd.Module() + mmd.set_mdversion(2) + mmd.set_name(name) + mmd.set_stream(stream) + mmd.set_version(int(version)) + mmd.set_context(context) + mmd.set_summary("foo") + mmd.set_description("foo") + licenses = Modulemd.SimpleSet() + licenses.add("GPL") + mmd.set_module_licenses(licenses) + + if not isinstance(requires_list, list): + requires_list = [requires_list] + if not isinstance(build_requires_list, list): + build_requires_list = [build_requires_list] + + deps_list = [] + for requires, build_requires in zip(requires_list, build_requires_list): + deps = Modulemd.Dependencies() + for req_name, req_streams in requires.items(): + deps.add_requires(req_name, req_streams) + for req_name, req_streams in build_requires.items(): + deps.add_buildrequires(req_name, req_streams) + deps_list.append(deps) + mmd.set_dependencies(deps_list) + + module_build = module_build_service.models.ModuleBuild() + module_build.name = name + module_build.stream = stream + module_build.version = version + module_build.state = models.BUILD_STATES['ready'] + module_build.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/unused.git?#ff1ea79' + module_build.batch = 1 + module_build.owner = 'Tom Brady' + module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18) + module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35) + module_build.rebuild_strategy = 'changed-and-after' + module_build.build_context = context + module_build.runtime_context = context + module_build.modulemd = mmd.dumps() + db.session.add(module_build) + db.session.commit() + + return module_build + + def _get_modules_build_required_by_module_recursively(self, module_build): + """ + Convenience wrapper around get_modules_build_required_by_module_recursively + returning the list with nsvc strings of modules returned by this the wrapped + method. + """ + modules = module_build_service.utils.get_modules_build_required_by_module_recursively( + db.session, module_build.mmd()) + nsvcs = [":".join([m.get_name(), m.get_stream(), str(m.get_version()), m.get_context()]) + for m in modules] + return nsvcs + + def _generate_default_modules(self): + """ + Generates gtk:1, gtk:2, foo:1 and foo:2 modules requiring the + platform:f28 and platform:f29 modules. + """ + self._make_module("gtk:1:0:c2", {"platform": ["f28"]}, {}) + self._make_module("gtk:1:0:c3", {"platform": ["f29"]}, {}) + self._make_module("gtk:2:0:c4", {"platform": ["f28"]}, {}) + self._make_module("gtk:2:0:c5", {"platform": ["f29"]}, {}) + self._make_module("foo:1:0:c2", {"platform": ["f28"]}, {}) + self._make_module("foo:1:0:c3", {"platform": ["f29"]}, {}) + self._make_module("foo:2:0:c4", {"platform": ["f28"]}, {}) + self._make_module("foo:2:0:c5", {"platform": ["f29"]}, {}) + self._make_module("platform:f28:0:c10", {}, {}) + self._make_module("platform:f29:0:c11", {}, {}) + + @pytest.mark.parametrize('requires,build_requires,expected', [ + ({}, {"gtk": ["1", "2"]}, + ['platform:f29:0:c11', 'gtk:2:0:c4', 'gtk:2:0:c5', + 'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']), + + ({}, {"gtk": ["1"], "foo": ["1"]}, + ['platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3', + 'foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11']), + + ({}, {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]}, + ['platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3', + 'foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11']), + + ([{}, {}], [{"gtk": ["1"], "foo": ["1"]}, {"gtk": ["2"], "foo": ["2"]}], + ['foo:1:0:c2', 'foo:1:0:c3', 'foo:2:0:c4', 'foo:2:0:c5', + 'platform:f28:0:c10', 'platform:f29:0:c11', 'gtk:1:0:c2', + 'gtk:1:0:c3', 'gtk:2:0:c4', 'gtk:2:0:c5']), + + ({}, {"gtk": ["-2"], "foo": ["-2"]}, + ['foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11', + 'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']), + + ({}, {"gtk": ["-1", "1"], "foo": ["-2", "1"]}, + ['foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11', + 'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']), + ]) + def test_get_required_modules_simple(self, requires, build_requires, expected): + module_build = self._make_module("app:1:0:c1", requires, build_requires) + self._generate_default_modules() + nsvcs = self._get_modules_build_required_by_module_recursively(module_build) + print nsvcs + assert set(nsvcs) == set(expected) + + def _generate_default_modules_recursion(self): + """ + Generates the gtk:1 module requiring foo:1 module requiring bar:1 + and lorem:1 modules which require base:f29 module requiring + platform:f29 module :). + """ + self._make_module("gtk:1:0:c2", {"foo": ["unknown"]}, {}) + self._make_module("gtk:1:1:c2", {"foo": ["1"]}, {}) + self._make_module("foo:1:0:c2", {"bar": ["unknown"]}, {}) + self._make_module("foo:1:1:c2", {"bar": ["1"], "lorem": ["1"]}, {}) + self._make_module("bar:1:0:c2", {"base": ["unknown"]}, {}) + self._make_module("bar:1:1:c2", {"base": ["f29"]}, {}) + self._make_module("lorem:1:0:c2", {"base": ["unknown"]}, {}) + self._make_module("lorem:1:1:c2", {"base": ["f29"]}, {}) + self._make_module("base:f29:0:c3", {"platform": ["f29"]}, {}) + self._make_module("platform:f29:0:c11", {}, {}) + + @pytest.mark.parametrize('requires,build_requires,expected', [ + ({}, {"gtk": ["1"]}, + ['foo:1:1:c2', 'base:f29:0:c3', 'platform:f29:0:c11', + 'bar:1:1:c2', 'gtk:1:1:c2', 'lorem:1:1:c2']), + + ({}, {"foo": ["1"]}, + ['foo:1:1:c2', 'base:f29:0:c3', 'platform:f29:0:c11', + 'bar:1:1:c2', 'lorem:1:1:c2']), + ]) + def test_get_required_modules_recursion(self, requires, build_requires, expected): + module_build = self._make_module("app:1:0:c1", requires, build_requires) + self._generate_default_modules_recursion() + nsvcs = self._get_modules_build_required_by_module_recursively(module_build) + print nsvcs + assert set(nsvcs) == set(expected)