diff --git a/docs/VIRTUAL_MODULES.rst b/docs/VIRTUAL_MODULES.rst index 46e6585c..f7f09376 100644 --- a/docs/VIRTUAL_MODULES.rst +++ b/docs/VIRTUAL_MODULES.rst @@ -72,6 +72,10 @@ Custom fields in xmd: module in the disttag of the RPMS being built. If the stream is not the appropriate value, then this can be overridden with a custom value using this property. This value can't contain a dash, since that is an invalid character in the disttag. +- ``default_modules_url`` - the URL to the list of modules, in the format of ``name:stream`` + separated by new lines, to include as default modules for any module that buildrequires this + module. Any default modules with conflicting streams will be ignored as well as any default module + not found in the MBS database. This field only applies to base modules. Virtual Streams diff --git a/module_build_service/scheduler/default_modules.py b/module_build_service/scheduler/default_modules.py new file mode 100644 index 00000000..450c7460 --- /dev/null +++ b/module_build_service/scheduler/default_modules.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import requests + +from module_build_service import conf, log, models +from module_build_service.errors import UnprocessableEntity +from module_build_service.utils.request_utils import requests_session +from module_build_service.resolver import system_resolver as resolver + + +def add_default_modules(db_session, mmd): + """ + Add default modules as buildrequires to the input modulemd. + + The base modules that are buildrequired can optionally link their default modules by specifying + a URL to a text file in xmd.mbs.default_modules_url. Any default module that isn't in the + database will be logged and ignored. + + :param db_session: a SQLAlchemy database session + :param Modulemd.ModuleStream mmd: the modulemd of the module to add the module defaults to + :raises RuntimeError: if the buildrequired base module isn't in the database or the default + modules list can't be downloaded + """ + log.info("Finding the default modules to include as buildrequires") + xmd = mmd.get_xmd() + buildrequires = xmd["mbs"]["buildrequires"] + + for module_name in conf.base_module_names: + bm_info = buildrequires.get(module_name) + if bm_info is None: + log.debug( + "The base module %s is not a buildrequire of the submitted module %s", + module_name, mmd.get_nsvc(), + ) + continue + + bm = models.ModuleBuild.get_build_from_nsvc( + db_session, module_name, bm_info["stream"], bm_info["version"], bm_info["context"], + ) + bm_nsvc = ":".join([ + module_name, bm_info["stream"], bm_info["version"], bm_info["context"], + ]) + if not bm: + raise RuntimeError("Failed to retrieve the module {} from the database".format(bm_nsvc)) + + bm_mmd = bm.mmd() + bm_xmd = bm_mmd.get_xmd() + default_modules_url = bm_xmd.get("mbs", {}).get("default_modules_url") + if not default_modules_url: + log.debug("The base module %s does not have any default modules", bm_nsvc) + continue + + try: + rv = requests_session.get(default_modules_url, timeout=10) + except requests.RequestException: + msg = ( + "The connection failed when getting the default modules associated with {}" + .format(bm_nsvc) + ) + log.exception(msg) + raise RuntimeError(msg) + + if not rv.ok: + log.error( + "The request to get the default modules associated with %s failed with the status " + 'code %d and error "%s"', + bm_nsvc, rv.status_code, rv.text, + ) + raise RuntimeError( + "Failed to retrieve the default modules for {}".format(bm_mmd.get_nsvc()) + ) + + default_modules = [m.strip() for m in rv.text.strip().split("\n")] + for default_module in default_modules: + try: + name, stream = default_module.split(":") + except ValueError: + log.error( + 'The default module "%s" from %s is in an invalid format', + default_module, rv.url, + ) + continue + + if name in buildrequires: + conflicting_stream = buildrequires[name]["stream"] + if stream == conflicting_stream: + log.info("The default module %s is already a buildrequire", default_module) + continue + + log.info( + "The default module %s will not be added as a buildrequire since %s:%s " + "is already a buildrequire", + default_module, name, conflicting_stream, + ) + continue + + try: + # We are reusing resolve_requires instead of directly querying the database since it + # provides the exact format that is needed for mbs.xmd.buildrequires. + # + # Only one default module is processed at a time in resolve_requires so that we + # are aware of which modules are not in the database, and can add those that are as + # buildrequires. + resolved = resolver.resolve_requires([default_module]) + except UnprocessableEntity: + log.warning( + "The default module %s from %s is not in the database and couldn't be added as " + "a buildrequire", + default_module, bm_nsvc + ) + continue + + nsvc = ":".join([name, stream, resolved[name]["version"], resolved[name]["context"]]) + log.info("Adding the default module %s as a buildrequire", nsvc) + buildrequires.update(resolved) + + mmd.set_xmd(xmd) diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index a480559b..472ce6d6 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -39,6 +39,7 @@ from module_build_service.utils import ( from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError from module_build_service.utils.ursine import handle_stream_collision_modules from module_build_service.utils.greenwave import greenwave +from module_build_service.scheduler.default_modules import add_default_modules from requests.exceptions import ConnectionError from module_build_service.utils import mmd_to_str @@ -163,6 +164,7 @@ def init(config, session, msg): failure_reason = "unspec" try: mmd = build.mmd() + add_default_modules(session, mmd) record_module_build_arches(mmd, build, session) record_component_builds(mmd, build, session=session) # The ursine.handle_stream_collision_modules is Koji specific. diff --git a/tests/test_scheduler/test_default_modules.py b/tests/test_scheduler/test_default_modules.py new file mode 100644 index 00000000..8d43d65d --- /dev/null +++ b/tests/test_scheduler/test_default_modules.py @@ -0,0 +1,145 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import textwrap + +from mock import patch +import pytest +import requests + +from module_build_service.models import ModuleBuild +from module_build_service.scheduler.default_modules import add_default_modules +from module_build_service.utils.general import load_mmd, mmd_to_str +from tests import clean_database, make_module, read_staged_data + + +@patch("module_build_service.scheduler.default_modules.requests_session") +def test_add_default_modules(mock_requests_session, db_session): + """ + Test that default modules present in the database are added, and the others are ignored. + """ + clean_database() + make_module(db_session, "python:3:12345:1") + make_module(db_session, "nodejs:11:2345:2") + mmd = load_mmd(read_staged_data("formatted_testmodule.yaml")) + xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"] + assert set(xmd_brs.keys()) == {"platform"} + + platform = ModuleBuild.get_build_from_nsvc( + db_session, + "platform", + xmd_brs["platform"]["stream"], + xmd_brs["platform"]["version"], + xmd_brs["platform"]["context"], + ) + assert platform + platform_mmd = platform.mmd() + platform_xmd = mmd.get_xmd() + default_modules_url = "http://domain.local/default_modules.txt" + platform_xmd["mbs"]["default_modules_url"] = default_modules_url + platform_mmd.set_xmd(platform_xmd) + platform.modulemd = mmd_to_str(platform_mmd) + db_session.commit() + + mock_requests_session.get.return_value.ok = True + # Also ensure that if there's an invalid line, it's just ignored + mock_requests_session.get.return_value.text = textwrap.dedent("""\ + nodejs:11 + python:3 + ruby:2.6 + some invalid stuff + """) + add_default_modules(db_session, mmd) + # Make sure that the default modules were added. ruby:2.6 will be ignored since it's not in + # the database + assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"nodejs", "platform", "python"} + mock_requests_session.get.assert_called_once_with(default_modules_url, timeout=10) + + +@patch("module_build_service.scheduler.default_modules.requests_session") +def test_add_default_modules_not_linked(mock_requests_session, db_session): + """ + Test that no default modules are added when they aren't linked from the base module. + """ + clean_database() + mmd = load_mmd(read_staged_data("formatted_testmodule.yaml")) + assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"platform"} + add_default_modules(db_session, mmd) + assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"platform"} + mock_requests_session.get.assert_not_called() + + +@patch("module_build_service.scheduler.default_modules.requests_session") +def test_add_default_modules_platform_not_available(mock_requests_session, db_session): + """ + Test that an exception is raised when the platform module that is buildrequired is missing. + + This error should never occur in practice. + """ + clean_database(False, False) + mmd = load_mmd(read_staged_data("formatted_testmodule.yaml")) + + expected_error = "Failed to retrieve the module platform:f28:3:00000000 from the database" + with pytest.raises(RuntimeError, match=expected_error): + add_default_modules(db_session, mmd) + + +@pytest.mark.parametrize("connection_error", (True, False)) +@patch("module_build_service.scheduler.default_modules.requests_session") +def test_add_default_modules_request_failed(mock_requests_session, connection_error, db_session): + """ + Test that an exception is raised when the request to get the default modules failed. + """ + clean_database() + make_module(db_session, "python:3:12345:1") + make_module(db_session, "nodejs:11:2345:2") + mmd = load_mmd(read_staged_data("formatted_testmodule.yaml")) + xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"] + assert set(xmd_brs.keys()) == {"platform"} + + platform = ModuleBuild.get_build_from_nsvc( + db_session, + "platform", + xmd_brs["platform"]["stream"], + xmd_brs["platform"]["version"], + xmd_brs["platform"]["context"], + ) + assert platform + platform_mmd = platform.mmd() + platform_xmd = mmd.get_xmd() + default_modules_url = "http://domain.local/default_modules.txt" + platform_xmd["mbs"]["default_modules_url"] = default_modules_url + platform_mmd.set_xmd(platform_xmd) + platform.modulemd = mmd_to_str(platform_mmd) + db_session.commit() + + if connection_error: + mock_requests_session.get.side_effect = requests.ConnectionError("some error") + expected_error = ( + "The connection failed when getting the default modules associated with " + "platform:f28:3:00000000" + ) + else: + mock_requests_session.get.return_value.ok = False + mock_requests_session.get.return_value.text = "some error" + expected_error = "Failed to retrieve the default modules for platform:f28:3:00000000" + + with pytest.raises(RuntimeError, match=expected_error): + add_default_modules(db_session, mmd)