From d3d1f2d3ddbb8a060f947a1599493c5bb0687889 Mon Sep 17 00:00:00 2001 From: mprahl Date: Fri, 3 Jan 2020 11:14:48 -0500 Subject: [PATCH] Split utils/mse.py This moves the code used by the backend and API to common/resolve.py and moves the code used just by the API to web/mse.py. --- module_build_service/common/resolve.py | 183 ++++++++++++++++++ .../scheduler/default_modules.py | 5 +- module_build_service/utils/__init__.py | 1 - module_build_service/utils/reuse.py | 2 +- module_build_service/utils/submit.py | 3 +- module_build_service/{utils => web}/mse.py | 173 +---------------- tests/test_common/test_resolve.py | 113 +++++++++++ tests/test_utils/test_utils.py | 4 +- tests/test_web/__init__.py | 0 .../test_mse.py} | 125 ++---------- 10 files changed, 317 insertions(+), 292 deletions(-) create mode 100644 module_build_service/common/resolve.py rename module_build_service/{utils => web}/mse.py (68%) create mode 100644 tests/test_common/test_resolve.py create mode 100644 tests/test_web/__init__.py rename tests/{test_utils/test_utils_mse.py => test_web/test_mse.py} (75%) diff --git a/module_build_service/common/resolve.py b/module_build_service/common/resolve.py new file mode 100644 index 00000000..2436c5f9 --- /dev/null +++ b/module_build_service/common/resolve.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from module_build_service import conf +from module_build_service.errors import StreamAmbigous +from module_build_service.resolver import GenericResolver + + +def expand_single_mse_streams( + db_session, name, streams, default_streams=None, raise_if_stream_ambigous=False): + """ + Helper method for `expand_mse_stream()` expanding single name:[streams]. + Returns list of expanded streams. + + :param db_session: SQLAlchemy DB session. + :param str name: Name of the module which will be expanded. + :param streams: List of streams to expand. + :type streams: list[str] + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + """ + from module_build_service import models + + default_streams = default_streams or {} + # Stream can be prefixed with '-' sign to define that this stream should + # not appear in a resulting list of streams. There can be two situations: + # a) all streams have '-' prefix. In this case, we treat list of streams + # as blacklist and we find all the valid streams and just remove those with + # '-' prefix. + # b) there is at least one stream without '-' prefix. In this case, we can + # ignore all the streams with '-' prefix and just add those without + # '-' prefix to the list of valid streams. + streams_is_blacklist = all(stream.startswith("-") for stream in streams) + if streams_is_blacklist or len(streams) == 0: + if name in default_streams: + expanded_streams = [default_streams[name]] + elif raise_if_stream_ambigous: + raise StreamAmbigous("There are multiple streams to choose from for module %s." % name) + else: + builds = models.ModuleBuild.get_last_build_in_all_streams(db_session, name) + expanded_streams = [build.stream for build in builds] + else: + expanded_streams = [] + for stream in streams: + if stream.startswith("-"): + if streams_is_blacklist and stream[1:] in expanded_streams: + expanded_streams.remove(stream[1:]) + else: + expanded_streams.append(stream) + + if len(expanded_streams) > 1: + if name in default_streams: + expanded_streams = [default_streams[name]] + elif raise_if_stream_ambigous: + raise StreamAmbigous( + "There are multiple streams %r to choose from for module %s." + % (expanded_streams, name) + ) + + return expanded_streams + + +def get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=None): + """ + Returns dict containing the base modules compatible with `base_mmd` grouped by their state. + + :param GenericResolver resolver: GenericResolver instance. + :param Modulemd base_mmd: Modulemd instant to return compatible modules for. + :param set ignore_ns: When set, defines the name:stream of base modules which will be ignored + by this function and therefore not returned. + :return dict: Dictionary with module's state name as a key and list of Modulemd objects for + each compatible base module in that state. For example: + { + "ready": [base_mmd_1, base_mmd_2] + "garbage": [base_mmd_3] + } + The input `base_mmd` is always included in the result in "ready" state. + """ + from module_build_service import models + + # Add the module to `seen` and `ret`. + ret = {"ready": [], "garbage": []} + ret["ready"].append(base_mmd) + ns = ":".join([base_mmd.get_module_name(), base_mmd.get_stream_name()]) + seen = set() if not ignore_ns else set(ignore_ns) + seen.add(ns) + + # Get the list of compatible virtual streams. + xmd = base_mmd.get_xmd() + virtual_streams = xmd.get("mbs", {}).get("virtual_streams") + + # In case there are no virtual_streams in the buildrequired name:stream, + # it is clear that there are no compatible streams, so return just this + # `base_mmd`. + if not virtual_streams: + return ret + + if conf.allow_only_compatible_base_modules: + stream_version_lte = True + states = ["ready"] + else: + stream_version_lte = False + states = ["ready", "garbage"] + + for state in states: + mmds = resolver.get_compatible_base_module_modulemds( + base_mmd, stream_version_lte, virtual_streams, + [models.BUILD_STATES[state]]) + ret_chunk = [] + # Add the returned mmds to the `seen` set to avoid querying those + # individually if they are part of the buildrequire streams for this + # base module. + for mmd_ in mmds: + mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) + # An extra precaution to ensure there are no duplicates. This can happen when there + # are two modules with the same name:stream - one in ready state and one in garbage + # state. + if mmd_ns not in seen: + ret_chunk.append(mmd_) + seen.add(mmd_ns) + ret[state] += ret_chunk + + return ret + + +def get_base_module_mmds(db_session, mmd): + """ + Returns list of MMDs of base modules buildrequired by `mmd` including the compatible + old versions of the base module based on the stream version. + + :param Modulemd mmd: Input modulemd metadata. + :rtype: dict with lists of Modulemd + :return: Dict with "ready" or "garbage" state name as a key and list of MMDs of base modules + buildrequired by `mmd` as a value. + """ + from module_build_service import models + + seen = set() + ret = {"ready": [], "garbage": []} + + resolver = GenericResolver.create(db_session, conf) + for deps in mmd.get_dependencies(): + buildrequires = { + module: deps.get_buildtime_streams(module) + for module in deps.get_buildtime_modules() + } + for name in conf.base_module_names: + if name not in buildrequires.keys(): + continue + + # Since the query below uses stream_version_lte, we can sort the streams by stream + # version in descending order to not perform unnecessary queries. Otherwise, if the + # streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be + # returned twice. Only one query for that scenario is necessary. + sorted_desc_streams = sorted( + buildrequires[name], reverse=True, key=models.ModuleBuild.get_stream_version) + for stream in sorted_desc_streams: + ns = ":".join([name, stream]) + if ns in seen: + continue + + # Get the MMD for particular buildrequired name:stream to find out the virtual + # streams according to which we can find the compatible streams later. + # The returned MMDs are all the module builds for name:stream in the highest + # version. Given the base module does not depend on other modules, it can appear + # only in single context and therefore the `mmds` should always contain just + # zero or one module build. + mmds = resolver.get_module_modulemds(name, stream) + if not mmds: + continue + base_mmd = mmds[0] + + new_ret = get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=seen) + for state in new_ret.keys(): + for mmd_ in new_ret[state]: + mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) + seen.add(mmd_ns) + ret[state] += new_ret[state] + break + return ret diff --git a/module_build_service/scheduler/default_modules.py b/module_build_service/scheduler/default_modules.py index e81ac7db..074bfed2 100644 --- a/module_build_service/scheduler/default_modules.py +++ b/module_build_service/scheduler/default_modules.py @@ -12,12 +12,13 @@ import six.moves.xmlrpc_client as xmlrpclib from module_build_service import conf, log, models, Modulemd, scm from module_build_service.common.koji import get_session, koji_retrying_multicall_map +from module_build_service.common.resolve import ( + expand_single_mse_streams, get_compatible_base_module_mmds +) from module_build_service.common.retry import retry from module_build_service.db_session import db_session from module_build_service.errors import UnprocessableEntity from module_build_service.resolver.base import GenericResolver -from module_build_service.utils.mse import ( - get_compatible_base_module_mmds, expand_single_mse_streams) def add_default_modules(mmd): diff --git a/module_build_service/utils/__init__.py b/module_build_service/utils/__init__.py index 4b880ac8..565b544e 100644 --- a/module_build_service/utils/__init__.py +++ b/module_build_service/utils/__init__.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT -from module_build_service.utils.mse import * # noqa from module_build_service.utils.views import * # noqa from module_build_service.utils.reuse import * # noqa from module_build_service.utils.submit import * # noqa diff --git a/module_build_service/utils/reuse.py b/module_build_service/utils/reuse.py index a9163b48..6e4672cc 100644 --- a/module_build_service/utils/reuse.py +++ b/module_build_service/utils/reuse.py @@ -6,7 +6,7 @@ from module_build_service import log, models, conf from module_build_service.db_session import db_session from module_build_service.resolver import GenericResolver from module_build_service.scheduler import events -from module_build_service.utils.mse import get_base_module_mmds +from module_build_service.common.resolve import get_base_module_mmds def reuse_component(component, previous_component_build, change_state_now=False, diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 0c141a59..b3317772 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -20,6 +20,7 @@ from module_build_service import conf, log, models, Modulemd from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str, to_text_type from module_build_service.db_session import db_session from module_build_service.errors import ValidationError, UnprocessableEntity, Forbidden, Conflict +from module_build_service.web.mse import generate_expanded_mmds from module_build_service.web.utils import deps_to_dict @@ -955,8 +956,6 @@ def submit_module_build(db_session, username, mmd, params): :rtype: list with ModuleBuild :return: List with submitted module builds. """ - from .mse import generate_expanded_mmds - log.debug( "Submitted %s module build for %s:%s:%s", ("scratch" if params.get("scratch", False) else "normal"), diff --git a/module_build_service/utils/mse.py b/module_build_service/web/mse.py similarity index 68% rename from module_build_service/utils/mse.py rename to module_build_service/web/mse.py index 420f9d3a..d5efdb92 100644 --- a/module_build_service/utils/mse.py +++ b/module_build_service/web/mse.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT from module_build_service import log, models, Modulemd, conf +from module_build_service.common.resolve import expand_single_mse_streams, get_base_module_mmds from module_build_service.common.utils import mmd_to_str from module_build_service.errors import StreamAmbigous from module_build_service.errors import UnprocessableEntity @@ -9,62 +10,6 @@ from module_build_service.web.utils import deps_to_dict from module_build_service.resolver import GenericResolver -def expand_single_mse_streams( - db_session, name, streams, default_streams=None, raise_if_stream_ambigous=False): - """ - Helper method for `expand_mse_stream()` expanding single name:[streams]. - Returns list of expanded streams. - - :param db_session: SQLAlchemy DB session. - :param str name: Name of the module which will be expanded. - :param streams: List of streams to expand. - :type streams: list[str] - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - """ - default_streams = default_streams or {} - # Stream can be prefixed with '-' sign to define that this stream should - # not appear in a resulting list of streams. There can be two situations: - # a) all streams have '-' prefix. In this case, we treat list of streams - # as blacklist and we find all the valid streams and just remove those with - # '-' prefix. - # b) there is at least one stream without '-' prefix. In this case, we can - # ignore all the streams with '-' prefix and just add those without - # '-' prefix to the list of valid streams. - streams_is_blacklist = all(stream.startswith("-") for stream in streams) - if streams_is_blacklist or len(streams) == 0: - if name in default_streams: - expanded_streams = [default_streams[name]] - elif raise_if_stream_ambigous: - raise StreamAmbigous("There are multiple streams to choose from for module %s." % name) - else: - builds = models.ModuleBuild.get_last_build_in_all_streams(db_session, name) - expanded_streams = [build.stream for build in builds] - else: - expanded_streams = [] - for stream in streams: - if stream.startswith("-"): - if streams_is_blacklist and stream[1:] in expanded_streams: - expanded_streams.remove(stream[1:]) - else: - expanded_streams.append(stream) - - if len(expanded_streams) > 1: - if name in default_streams: - expanded_streams = [default_streams[name]] - elif raise_if_stream_ambigous: - raise StreamAmbigous( - "There are multiple streams %r to choose from for module %s." - % (expanded_streams, name) - ) - - return expanded_streams - - def expand_mse_streams(db_session, mmd, default_streams=None, raise_if_stream_ambigous=False): """ Expands streams in both buildrequires/requires sections of MMD. @@ -186,122 +131,6 @@ def _get_mmds_from_requires( return mmds -def get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=None): - """ - Returns dict containing the base modules compatible with `base_mmd` grouped by their state. - - :param GenericResolver resolver: GenericResolver instance. - :param Modulemd base_mmd: Modulemd instant to return compatible modules for. - :param set ignore_ns: When set, defines the name:stream of base modules which will be ignored - by this function and therefore not returned. - :return dict: Dictionary with module's state name as a key and list of Modulemd objects for - each compatible base module in that state. For example: - { - "ready": [base_mmd_1, base_mmd_2] - "garbage": [base_mmd_3] - } - The input `base_mmd` is always included in the result in "ready" state. - """ - # Add the module to `seen` and `ret`. - ret = {"ready": [], "garbage": []} - ret["ready"].append(base_mmd) - ns = ":".join([base_mmd.get_module_name(), base_mmd.get_stream_name()]) - seen = set() if not ignore_ns else set(ignore_ns) - seen.add(ns) - - # Get the list of compatible virtual streams. - xmd = base_mmd.get_xmd() - virtual_streams = xmd.get("mbs", {}).get("virtual_streams") - - # In case there are no virtual_streams in the buildrequired name:stream, - # it is clear that there are no compatible streams, so return just this - # `base_mmd`. - if not virtual_streams: - return ret - - if conf.allow_only_compatible_base_modules: - stream_version_lte = True - states = ["ready"] - else: - stream_version_lte = False - states = ["ready", "garbage"] - - for state in states: - mmds = resolver.get_compatible_base_module_modulemds( - base_mmd, stream_version_lte, virtual_streams, - [models.BUILD_STATES[state]]) - ret_chunk = [] - # Add the returned mmds to the `seen` set to avoid querying those - # individually if they are part of the buildrequire streams for this - # base module. - for mmd_ in mmds: - mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) - # An extra precaution to ensure there are no duplicates. This can happen when there - # are two modules with the same name:stream - one in ready state and one in garbage - # state. - if mmd_ns not in seen: - ret_chunk.append(mmd_) - seen.add(mmd_ns) - ret[state] += ret_chunk - - return ret - - -def get_base_module_mmds(db_session, mmd): - """ - Returns list of MMDs of base modules buildrequired by `mmd` including the compatible - old versions of the base module based on the stream version. - - :param Modulemd mmd: Input modulemd metadata. - :rtype: dict with lists of Modulemd - :return: Dict with "ready" or "garbage" state name as a key and list of MMDs of base modules - buildrequired by `mmd` as a value. - """ - seen = set() - ret = {"ready": [], "garbage": []} - - resolver = GenericResolver.create(db_session, conf) - for deps in mmd.get_dependencies(): - buildrequires = { - module: deps.get_buildtime_streams(module) - for module in deps.get_buildtime_modules() - } - for name in conf.base_module_names: - if name not in buildrequires.keys(): - continue - - # Since the query below uses stream_version_lte, we can sort the streams by stream - # version in descending order to not perform unnecessary queries. Otherwise, if the - # streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be - # returned twice. Only one query for that scenario is necessary. - sorted_desc_streams = sorted( - buildrequires[name], reverse=True, key=models.ModuleBuild.get_stream_version) - for stream in sorted_desc_streams: - ns = ":".join([name, stream]) - if ns in seen: - continue - - # Get the MMD for particular buildrequired name:stream to find out the virtual - # streams according to which we can find the compatible streams later. - # The returned MMDs are all the module builds for name:stream in the highest - # version. Given the base module does not depend on other modules, it can appear - # only in single context and therefore the `mmds` should always contain just - # zero or one module build. - mmds = resolver.get_module_modulemds(name, stream) - if not mmds: - continue - base_mmd = mmds[0] - - new_ret = get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=seen) - for state in new_ret.keys(): - for mmd_ in new_ret[state]: - mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()]) - seen.add(mmd_ns) - ret[state] += new_ret[state] - break - return ret - - def get_mmds_required_by_module_recursively( db_session, mmd, default_streams=None, raise_if_stream_ambigous=False ): diff --git a/tests/test_common/test_resolve.py b/tests/test_common/test_resolve.py new file mode 100644 index 00000000..efd62777 --- /dev/null +++ b/tests/test_common/test_resolve.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from mock import patch, PropertyMock +import pytest + +from module_build_service.common.utils import load_mmd +from module_build_service.common.resolve import get_base_module_mmds +from module_build_service import Modulemd, models +from module_build_service.db_session import db_session +from tests import clean_database, make_module_in_db, init_data, read_staged_data + + +class TestResolve: + def setup_method(self, test_method): + clean_database(False) + + def teardown_method(self, test_method): + clean_database() + + def test__get_base_module_mmds(self): + """Ensure the correct results are returned without duplicates.""" + init_data(data_size=1, multiple_stream_versions=True) + mmd = load_mmd(read_staged_data("testmodule_v2.yaml")) + deps = mmd.get_dependencies()[0] + new_deps = Modulemd.Dependencies() + for stream in deps.get_runtime_streams("platform"): + new_deps.add_runtime_stream("platform", stream) + new_deps.add_buildtime_stream("platform", "f29.1.0") + new_deps.add_buildtime_stream("platform", "f29.2.0") + mmd.remove_dependencies(deps) + mmd.add_dependencies(new_deps) + + mmds = get_base_module_mmds(db_session, mmd) + expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} + # Verify no duplicates were returned before doing set operations + assert len(mmds["ready"]) == len(expected) + # Verify the expected ones were returned + actual = set() + for mmd_ in mmds["ready"]: + actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) + assert actual == expected + + @pytest.mark.parametrize("virtual_streams", (None, ["f29"], ["lp29"])) + def test__get_base_module_mmds_virtual_streams(self, virtual_streams): + """Ensure the correct results are returned without duplicates.""" + init_data(data_size=1, multiple_stream_versions=True) + mmd = load_mmd(read_staged_data("testmodule_v2")) + deps = mmd.get_dependencies()[0] + new_deps = Modulemd.Dependencies() + for stream in deps.get_runtime_streams("platform"): + new_deps.add_runtime_stream("platform", stream) + new_deps.add_buildtime_stream("platform", "f29.2.0") + mmd.remove_dependencies(deps) + mmd.add_dependencies(new_deps) + + make_module_in_db("platform:lp29.1.1:12:c11", virtual_streams=virtual_streams) + + mmds = get_base_module_mmds(db_session, mmd) + if virtual_streams == ["f29"]: + expected = { + "platform:f29.0.0", + "platform:f29.1.0", + "platform:f29.2.0", + "platform:lp29.1.1" + } + else: + expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} + # Verify no duplicates were returned before doing set operations + assert len(mmds["ready"]) == len(expected) + # Verify the expected ones were returned + actual = set() + for mmd_ in mmds["ready"]: + actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) + assert actual == expected + + @patch( + "module_build_service.config.Config.allow_only_compatible_base_modules", + new_callable=PropertyMock, return_value=False + ) + def test__get_base_module_mmds_virtual_streams_only_major_versions(self, cfg): + """Ensure the correct results are returned without duplicates.""" + init_data(data_size=1, multiple_stream_versions=["foo28", "foo29", "foo30"]) + + # Mark platform:foo28 as garbage to test that it is still considered as compatible. + platform = db_session.query(models.ModuleBuild).filter_by( + name="platform", stream="foo28").first() + platform.state = "garbage" + db_session.add(platform) + db_session.commit() + + mmd = load_mmd(read_staged_data("testmodule_v2")) + deps = mmd.get_dependencies()[0] + new_deps = Modulemd.Dependencies() + for stream in deps.get_runtime_streams("platform"): + new_deps.add_runtime_stream("platform", stream) + new_deps.add_buildtime_stream("platform", "foo29") + mmd.remove_dependencies(deps) + mmd.add_dependencies(new_deps) + + mmds = get_base_module_mmds(db_session, mmd) + expected = {} + expected["ready"] = {"platform:foo29", "platform:foo30"} + expected["garbage"] = {"platform:foo28"} + + # Verify no duplicates were returned before doing set operations + assert len(mmds) == len(expected) + for k in expected.keys(): + assert len(mmds[k]) == len(expected[k]) + # Verify the expected ones were returned + actual = set() + for mmd_ in mmds[k]: + actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) + assert actual == expected[k] diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index b6239753..66c19d01 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -703,7 +703,7 @@ class TestUtils: v = module_build_service.utils.submit.get_prefixed_version(mmd) assert v == 7000120180205135154 - @patch("module_build_service.utils.mse.generate_expanded_mmds") + @patch("module_build_service.utils.submit.generate_expanded_mmds") def test_submit_build_new_mse_build(self, generate_expanded_mmds): """ Tests that finished build can be resubmitted in case the resubmitted @@ -729,7 +729,7 @@ class TestUtils: assert builds[0].siblings(db_session) == [builds[1].id] assert builds[1].siblings(db_session) == [builds[0].id] - @patch("module_build_service.utils.mse.generate_expanded_mmds") + @patch("module_build_service.utils.submit.generate_expanded_mmds") @patch( "module_build_service.config.Config.scratch_build_only_branches", new_callable=mock.PropertyMock, diff --git a/tests/test_web/__init__.py b/tests/test_web/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_utils/test_utils_mse.py b/tests/test_web/test_mse.py similarity index 75% rename from tests/test_utils/test_utils_mse.py rename to tests/test_web/test_mse.py index 094b36cd..d2729fbb 100644 --- a/tests/test_utils/test_utils_mse.py +++ b/tests/test_web/test_mse.py @@ -1,17 +1,16 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT -from mock import patch, PropertyMock import pytest -import module_build_service.utils -from module_build_service.common.utils import load_mmd -from module_build_service import Modulemd, models from module_build_service.db_session import db_session from module_build_service.errors import StreamAmbigous -from tests import clean_database, make_module_in_db, init_data, read_staged_data +from module_build_service.web.mse import ( + expand_mse_streams, generate_expanded_mmds, get_mmds_required_by_module_recursively +) +from tests import clean_database, make_module_in_db -class TestUtilsModuleStreamExpansion: +class TestModuleStreamExpansion: def setup_method(self, test_method): clean_database(False) @@ -25,9 +24,8 @@ class TestUtilsModuleStreamExpansion: method. """ mmd = module_build.mmd() - module_build_service.utils.expand_mse_streams(db_session, mmd) - modules = module_build_service.utils.get_mmds_required_by_module_recursively( - db_session, mmd) + expand_mse_streams(db_session, mmd) + modules = get_mmds_required_by_module_recursively(db_session, mmd) nsvcs = [m.get_nsvc() for m in modules] return nsvcs @@ -67,8 +65,7 @@ class TestUtilsModuleStreamExpansion: }, }], ) - mmds = module_build_service.utils.generate_expanded_mmds( - db_session, module_build.mmd()) + mmds = generate_expanded_mmds(db_session, module_build.mmd()) contexts = {mmd.get_context() for mmd in mmds} assert {"e1e005fb", "ce132a1e"} == contexts @@ -168,11 +165,10 @@ class TestUtilsModuleStreamExpansion: # and also that it does not raise an exception otherwise. if stream_ambigous: with pytest.raises(StreamAmbigous): - module_build_service.utils.generate_expanded_mmds( + generate_expanded_mmds( db_session, module_build.mmd(), raise_if_stream_ambigous=True) else: - module_build_service.utils.generate_expanded_mmds( - db_session, module_build.mmd(), raise_if_stream_ambigous=True) + generate_expanded_mmds(db_session, module_build.mmd(), raise_if_stream_ambigous=True) # Check that if stream is ambigous and we define the stream, it does not raise # an exception. @@ -181,14 +177,14 @@ class TestUtilsModuleStreamExpansion: for ns in list(expected_buildrequires)[0]: name, stream = ns.split(":") default_streams[name] = stream - module_build_service.utils.generate_expanded_mmds( + generate_expanded_mmds( db_session, module_build.mmd(), raise_if_stream_ambigous=True, default_streams=default_streams, ) - mmds = module_build_service.utils.generate_expanded_mmds(db_session, module_build.mmd()) + mmds = generate_expanded_mmds(db_session, module_build.mmd()) buildrequires_per_mmd_xmd = set() buildrequires_per_mmd_buildrequires = set() @@ -255,7 +251,7 @@ class TestUtilsModuleStreamExpansion: def test_generate_expanded_mmds_requires(self, module_deps, expected): self._generate_default_modules() module_build = make_module_in_db("app:1:0:c1", module_deps) - mmds = module_build_service.utils.generate_expanded_mmds(db_session, module_build.mmd()) + mmds = generate_expanded_mmds(db_session, module_build.mmd()) requires_per_mmd = set() for mmd in mmds: @@ -467,98 +463,3 @@ class TestUtilsModuleStreamExpansion: self._generate_default_modules_modules_multiple_stream_versions() nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session) assert set(nsvcs) == set(expected) - - def test__get_base_module_mmds(self): - """Ensure the correct results are returned without duplicates.""" - init_data(data_size=1, multiple_stream_versions=True) - mmd = load_mmd(read_staged_data("testmodule_v2.yaml")) - deps = mmd.get_dependencies()[0] - new_deps = Modulemd.Dependencies() - for stream in deps.get_runtime_streams("platform"): - new_deps.add_runtime_stream("platform", stream) - new_deps.add_buildtime_stream("platform", "f29.1.0") - new_deps.add_buildtime_stream("platform", "f29.2.0") - mmd.remove_dependencies(deps) - mmd.add_dependencies(new_deps) - - mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd) - expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} - # Verify no duplicates were returned before doing set operations - assert len(mmds["ready"]) == len(expected) - # Verify the expected ones were returned - actual = set() - for mmd_ in mmds["ready"]: - actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) - assert actual == expected - - @pytest.mark.parametrize("virtual_streams", (None, ["f29"], ["lp29"])) - def test__get_base_module_mmds_virtual_streams(self, virtual_streams): - """Ensure the correct results are returned without duplicates.""" - init_data(data_size=1, multiple_stream_versions=True) - mmd = load_mmd(read_staged_data("testmodule_v2")) - deps = mmd.get_dependencies()[0] - new_deps = Modulemd.Dependencies() - for stream in deps.get_runtime_streams("platform"): - new_deps.add_runtime_stream("platform", stream) - new_deps.add_buildtime_stream("platform", "f29.2.0") - mmd.remove_dependencies(deps) - mmd.add_dependencies(new_deps) - - make_module_in_db("platform:lp29.1.1:12:c11", virtual_streams=virtual_streams) - - mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd) - if virtual_streams == ["f29"]: - expected = { - "platform:f29.0.0", - "platform:f29.1.0", - "platform:f29.2.0", - "platform:lp29.1.1" - } - else: - expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"} - # Verify no duplicates were returned before doing set operations - assert len(mmds["ready"]) == len(expected) - # Verify the expected ones were returned - actual = set() - for mmd_ in mmds["ready"]: - actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) - assert actual == expected - - @patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", - new_callable=PropertyMock, return_value=False - ) - def test__get_base_module_mmds_virtual_streams_only_major_versions(self, cfg): - """Ensure the correct results are returned without duplicates.""" - init_data(data_size=1, multiple_stream_versions=["foo28", "foo29", "foo30"]) - - # Mark platform:foo28 as garbage to test that it is still considered as compatible. - platform = db_session.query(models.ModuleBuild).filter_by( - name="platform", stream="foo28").first() - platform.state = "garbage" - db_session.add(platform) - db_session.commit() - - mmd = load_mmd(read_staged_data("testmodule_v2")) - deps = mmd.get_dependencies()[0] - new_deps = Modulemd.Dependencies() - for stream in deps.get_runtime_streams("platform"): - new_deps.add_runtime_stream("platform", stream) - new_deps.add_buildtime_stream("platform", "foo29") - mmd.remove_dependencies(deps) - mmd.add_dependencies(new_deps) - - mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd) - expected = {} - expected["ready"] = {"platform:foo29", "platform:foo30"} - expected["garbage"] = {"platform:foo28"} - - # Verify no duplicates were returned before doing set operations - assert len(mmds) == len(expected) - for k in expected.keys(): - assert len(mmds[k]) == len(expected[k]) - # Verify the expected ones were returned - actual = set() - for mmd_ in mmds[k]: - actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) - assert actual == expected[k]