modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-03 02:37:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add the ability to automatically buildrequire default modules defined by the buildrequired base module

Browse Source 
A base module can set xmd.mbs.default_modules_url, which contains a
URL to a list of modules in the format of name:stream separated by
new lines. When a module buildrequires this base module, the list
of default modules are added as buildrequires of the module automatically
unless there are conflicting streams or the default module is not
in the MBS database.
This commit is contained in:
mprahl
2019-07-03 16:18:28 -04:00
parent 9f55ce724d
commit 96d44d049e
4 changed files with 286 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/VIRTUAL_MODULES.rst
View File

@@ -72,6 +72,10 @@ Custom fields in xmd:
module in the disttag of the RPMS being built. If the stream is not the appropriate value, then
this can be overridden with a custom value using this property. This value can't contain a dash,
since that is an invalid character in the disttag.
- ``default_modules_url`` - the URL to the list of modules, in the format of ``name:stream``
separated by new lines, to include as default modules for any module that buildrequires this
module. Any default modules with conflicting streams will be ignored as well as any default module
not found in the MBS database. This field only applies to base modules.
Virtual Streams

135
module_build_service/scheduler/default_modules.py Normal file
View File

@@ -0,0 +1,135 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import requests
from module_build_service import conf, log, models
from module_build_service.errors import UnprocessableEntity
from module_build_service.utils.request_utils import requests_session
from module_build_service.resolver import system_resolver as resolver
def add_default_modules(db_session, mmd):
"""
Add default modules as buildrequires to the input modulemd.
The base modules that are buildrequired can optionally link their default modules by specifying
a URL to a text file in xmd.mbs.default_modules_url. Any default module that isn't in the
database will be logged and ignored.
:param db_session: a SQLAlchemy database session
:param Modulemd.ModuleStream mmd: the modulemd of the module to add the module defaults to
:raises RuntimeError: if the buildrequired base module isn't in the database or the default
modules list can't be downloaded
"""
log.info("Finding the default modules to include as buildrequires")
xmd = mmd.get_xmd()
buildrequires = xmd["mbs"]["buildrequires"]
for module_name in conf.base_module_names:
bm_info = buildrequires.get(module_name)
if bm_info is None:
log.debug(
"The base module %s is not a buildrequire of the submitted module %s",
module_name, mmd.get_nsvc(),
)
continue
bm = models.ModuleBuild.get_build_from_nsvc(
db_session, module_name, bm_info["stream"], bm_info["version"], bm_info["context"],
)
bm_nsvc = ":".join([
module_name, bm_info["stream"], bm_info["version"], bm_info["context"],
])
if not bm:
raise RuntimeError("Failed to retrieve the module {} from the database".format(bm_nsvc))
bm_mmd = bm.mmd()
bm_xmd = bm_mmd.get_xmd()
default_modules_url = bm_xmd.get("mbs", {}).get("default_modules_url")
if not default_modules_url:
log.debug("The base module %s does not have any default modules", bm_nsvc)
continue
try:
rv = requests_session.get(default_modules_url, timeout=10)
except requests.RequestException:
msg = (
"The connection failed when getting the default modules associated with {}"
.format(bm_nsvc)
)
log.exception(msg)
raise RuntimeError(msg)
if not rv.ok:
log.error(
"The request to get the default modules associated with %s failed with the status "
'code %d and error "%s"',
bm_nsvc, rv.status_code, rv.text,
)
raise RuntimeError(
"Failed to retrieve the default modules for {}".format(bm_mmd.get_nsvc())
)
default_modules = [m.strip() for m in rv.text.strip().split("\n")]
for default_module in default_modules:
try:
name, stream = default_module.split(":")
except ValueError:
log.error(
'The default module "%s" from %s is in an invalid format',
default_module, rv.url,
)
continue
if name in buildrequires:
conflicting_stream = buildrequires[name]["stream"]
if stream == conflicting_stream:
log.info("The default module %s is already a buildrequire", default_module)
continue
log.info(
"The default module %s will not be added as a buildrequire since %s:%s "
"is already a buildrequire",
default_module, name, conflicting_stream,
)
continue
try:
# We are reusing resolve_requires instead of directly querying the database since it
# provides the exact format that is needed for mbs.xmd.buildrequires.
#
# Only one default module is processed at a time in resolve_requires so that we
# are aware of which modules are not in the database, and can add those that are as
# buildrequires.
resolved = resolver.resolve_requires([default_module])
except UnprocessableEntity:
log.warning(
"The default module %s from %s is not in the database and couldn't be added as "
"a buildrequire",
default_module, bm_nsvc
)
continue
nsvc = ":".join([name, stream, resolved[name]["version"], resolved[name]["context"]])
log.info("Adding the default module %s as a buildrequire", nsvc)
buildrequires.update(resolved)
mmd.set_xmd(xmd)

2
module_build_service/scheduler/handlers/modules.py
View File

@@ -39,6 +39,7 @@ from module_build_service.utils import (
from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError
from module_build_service.utils.ursine import handle_stream_collision_modules
from module_build_service.utils.greenwave import greenwave
from module_build_service.scheduler.default_modules import add_default_modules
from requests.exceptions import ConnectionError
from module_build_service.utils import mmd_to_str
@@ -163,6 +164,7 @@ def init(config, session, msg):
failure_reason = "unspec"
try:
mmd = build.mmd()
add_default_modules(session, mmd)
record_module_build_arches(mmd, build, session)
record_component_builds(mmd, build, session=session)
# The ursine.handle_stream_collision_modules is Koji specific.

145
tests/test_scheduler/test_default_modules.py Normal file
View File

@@ -0,0 +1,145 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import textwrap
from mock import patch
import pytest
import requests
from module_build_service.models import ModuleBuild
from module_build_service.scheduler.default_modules import add_default_modules
from module_build_service.utils.general import load_mmd, mmd_to_str
from tests import clean_database, make_module, read_staged_data
@patch("module_build_service.scheduler.default_modules.requests_session")
def test_add_default_modules(mock_requests_session, db_session):
"""
Test that default modules present in the database are added, and the others are ignored.
"""
clean_database()
make_module(db_session, "python:3:12345:1")
make_module(db_session, "nodejs:11:2345:2")
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]
assert set(xmd_brs.keys()) == {"platform"}
platform = ModuleBuild.get_build_from_nsvc(
db_session,
"platform",
xmd_brs["platform"]["stream"],
xmd_brs["platform"]["version"],
xmd_brs["platform"]["context"],
)
assert platform
platform_mmd = platform.mmd()
platform_xmd = mmd.get_xmd()
default_modules_url = "http://domain.local/default_modules.txt"
platform_xmd["mbs"]["default_modules_url"] = default_modules_url
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
db_session.commit()
mock_requests_session.get.return_value.ok = True
# Also ensure that if there's an invalid line, it's just ignored
mock_requests_session.get.return_value.text = textwrap.dedent("""\
nodejs:11
python:3
ruby:2.6
some invalid stuff
""")
add_default_modules(db_session, mmd)
# Make sure that the default modules were added. ruby:2.6 will be ignored since it's not in
# the database
assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"nodejs", "platform", "python"}
mock_requests_session.get.assert_called_once_with(default_modules_url, timeout=10)
@patch("module_build_service.scheduler.default_modules.requests_session")
def test_add_default_modules_not_linked(mock_requests_session, db_session):
"""
Test that no default modules are added when they aren't linked from the base module.
"""
clean_database()
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"platform"}
add_default_modules(db_session, mmd)
assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"platform"}
mock_requests_session.get.assert_not_called()
@patch("module_build_service.scheduler.default_modules.requests_session")
def test_add_default_modules_platform_not_available(mock_requests_session, db_session):
"""
Test that an exception is raised when the platform module that is buildrequired is missing.
This error should never occur in practice.
"""
clean_database(False, False)
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
expected_error = "Failed to retrieve the module platform:f28:3:00000000 from the database"
with pytest.raises(RuntimeError, match=expected_error):
add_default_modules(db_session, mmd)
@pytest.mark.parametrize("connection_error", (True, False))
@patch("module_build_service.scheduler.default_modules.requests_session")
def test_add_default_modules_request_failed(mock_requests_session, connection_error, db_session):
"""
Test that an exception is raised when the request to get the default modules failed.
"""
clean_database()
make_module(db_session, "python:3:12345:1")
make_module(db_session, "nodejs:11:2345:2")
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]
assert set(xmd_brs.keys()) == {"platform"}
platform = ModuleBuild.get_build_from_nsvc(
db_session,
"platform",
xmd_brs["platform"]["stream"],
xmd_brs["platform"]["version"],
xmd_brs["platform"]["context"],
)
assert platform
platform_mmd = platform.mmd()
platform_xmd = mmd.get_xmd()
default_modules_url = "http://domain.local/default_modules.txt"
platform_xmd["mbs"]["default_modules_url"] = default_modules_url
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
db_session.commit()
if connection_error:
mock_requests_session.get.side_effect = requests.ConnectionError("some error")
expected_error = (
"The connection failed when getting the default modules associated with "
"platform:f28:3:00000000"
)
else:
mock_requests_session.get.return_value.ok = False
mock_requests_session.get.return_value.text = "some error"
expected_error = "Failed to retrieve the default modules for platform:f28:3:00000000"
with pytest.raises(RuntimeError, match=expected_error):
add_default_modules(db_session, mmd)