diff --git a/module_build_service/common/scm.py b/module_build_service/common/scm.py index c8b1d79a..cfb6429d 100644 --- a/module_build_service/common/scm.py +++ b/module_build_service/common/scm.py @@ -3,7 +3,6 @@ """SCM handler functions.""" from __future__ import absolute_import -import datetime import os import subprocess as sp import re @@ -18,6 +17,7 @@ from module_build_service.common.errors import ( ProgrammingError, ) from module_build_service.common.retry import retry +from module_build_service.common.utils import provide_module_stream_version_from_timestamp def scm_url_schemes(terse=False): @@ -229,8 +229,7 @@ class SCM(object): raise timestamp = SCM._run(["git", "show", "-s", "--format=%ct"], chdir=self.sourcedir)[1] - dt = datetime.datetime.utcfromtimestamp(int(timestamp)) - self.version = dt.strftime("%Y%m%d%H%M%S") + self.version = provide_module_stream_version_from_timestamp(timestamp=timestamp) else: raise RuntimeError("checkout: Unhandled SCM scheme.") return self.sourcedir diff --git a/module_build_service/common/submit.py b/module_build_service/common/submit.py index b0b78494..e02f00de 100644 --- a/module_build_service/common/submit.py +++ b/module_build_service/common/submit.py @@ -88,12 +88,10 @@ def fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False, mand # If the version is in the modulemd, throw an exception since the version # since the version is generated by MBS - if mmd.get_version(): + if mmd.get_mdversion() == 2 and mmd.get_version(): raise ValidationError( 'The version "{0}" is already defined in the modulemd but it shouldn\'t be since the ' "version is generated based on the commit time".format(mmd.get_version()) ) - else: - mmd.set_version(int(scm.version)) return mmd, scm diff --git a/module_build_service/common/utils.py b/module_build_service/common/utils.py index b80b829d..ea021fcd 100644 --- a/module_build_service/common/utils.py +++ b/module_build_service/common/utils.py @@ -4,6 +4,7 @@ from __future__ import absolute_import from datetime import datetime from functools import partial import os +import time from gi.repository.GLib import Error as ModuleMDError from six import string_types, text_type @@ -27,44 +28,37 @@ def load_mmd(yaml, is_file=False): if not yaml: raise UnprocessableEntity('The input modulemd was empty') - target_mmd_version = Modulemd.ModuleStreamVersionEnum.TWO try: if is_file: - mmd = Modulemd.ModuleStream.read_file(yaml, True) + mmd = Modulemd.read_packager_file(yaml) else: - mmd = Modulemd.ModuleStream.read_string(to_text_type(yaml), True) - mmd.validate() - if mmd.get_mdversion() < target_mmd_version: - mmd = mmd.upgrade(target_mmd_version) - elif mmd.get_mdversion() > target_mmd_version: - log.error("Encountered a modulemd file with the version %d", mmd.get_mdversion()) - raise UnprocessableEntity( - "The modulemd version cannot be greater than {}".format(target_mmd_version)) + mmd = Modulemd.read_packager_string(to_text_type(yaml)) + + mmd_version = mmd.get_mdversion() + + # both legacy packager v1 and stream v1 are directly upgraded to v2 + if mmd_version == 1: + mmd = mmd.upgrade(2) + except ModuleMDError as e: - not_found = False + error = None if is_file: - error = "The modulemd {} is invalid.".format(os.path.basename(yaml)) if os.path.exists(yaml): with open(yaml, "rt") as yaml_hdl: - log.debug("Modulemd that failed to load:\n%s", yaml_hdl.read()) + log.debug("The modulemd file '%s' failed to load:\n%s", + yaml, + yaml_hdl.read()) else: - not_found = True - error = "The modulemd file {} was not found.".format(os.path.basename(yaml)) + error = "The modulemd file {} was not found.".format(yaml) log.error("The modulemd file %s was not found.", yaml) - else: - error = "The modulemd is invalid." - log.debug("Modulemd that failed to load:\n%s", yaml) - if "modulemd-error-quark: " in str(e): - error = "{} The error was '{}'.".format( - error, str(e).split("modulemd-error-quark: ")[-1]) - elif "Unknown ModuleStream version" in str(e): - error = ( - "{}. The modulemd version can't be greater than {}." - .format(error, target_mmd_version) - ) - elif not_found is False: - error = "{} Please verify the syntax is correct.".format(error) + if not error: + if is_file: + error = ("The modulemd {} is invalid. The error was:\n'{}'\nPlease verify the" + " syntax is correct.").format(os.path.basename(yaml), str(e)) + else: + error = ("The modulemd is invalid. The error was:\n'{}'\nPlease verify the" + " syntax is correct.").format(str(e)) log.exception(error) raise UnprocessableEntity(error) @@ -226,3 +220,39 @@ def mmd_to_str(mmd): index = Modulemd.ModuleIndex() index.add_module_stream(mmd) return to_text_type(index.dump_to_string()) + + +def provide_module_stream_version_from_timestamp(timestamp=None): + """ + Provides a module stream version from a unix timestamp. If the timestamp is not defined + it will will generate one.. + + :param timestamp: modulemd object representing module metadata. + :type mmd: Modulemd.Module + :return: module stream version + :rtype: int + """ + if timestamp: + dt = datetime.utcfromtimestamp(int(timestamp)) + else: + dt = datetime.utcfromtimestamp(int(time.time())) + + return int(dt.strftime("%Y%m%d%H%M%S")) + + +def provide_module_stream_version_from_mmd(mmd): + """ + Provides a module stream version for a module stream. If a mmd v2 already contains + a version it will return it else it will generate a new one. + + :param mmd: modulemd object representing module metadata. + :type mmd: Modulemd.Module + :return: module stream version + :rtype: int + """ + + if mmd.get_mdversion() == 2 and mmd.get_version(): + return mmd.get_version() + + dt = datetime.utcfromtimestamp(int(time.time())) + return int(dt.strftime("%Y%m%d%H%M%S")) diff --git a/module_build_service/web/submit.py b/module_build_service/web/submit.py index f3ab15ec..3dc59e5a 100644 --- a/module_build_service/web/submit.py +++ b/module_build_service/web/submit.py @@ -6,7 +6,6 @@ from datetime import datetime import math import os import re -import time from gi.repository import GLib import requests @@ -16,7 +15,8 @@ from module_build_service.common.errors import Conflict, Forbidden, ValidationEr from module_build_service.common.messaging import notify_on_module_state_change from module_build_service.common.modulemd import Modulemd from module_build_service.common.submit import fetch_mmd -from module_build_service.common.utils import load_mmd, mmd_to_str, to_text_type +from module_build_service.common.utils import (load_mmd, mmd_to_str, to_text_type, + provide_module_stream_version_from_mmd) from module_build_service.web.mse import generate_expanded_mmds, generate_mmds_from_static_contexts from module_build_service.web.utils import deps_to_dict @@ -47,7 +47,7 @@ def validate_mmd(mmd): if name not in conf.allowed_privileged_module_names: raise ValidationError('The "mbs" xmd field is reserved for MBS') - allowed_keys = ["disttag_marking", "koji_tag_arches"] + allowed_keys = ["disttag_marking", "koji_tag_arches", 'static_context'] for key in xmd["mbs"].keys(): if key not in allowed_keys: raise ValidationError('The "mbs" xmd field is reserved for MBS') @@ -112,7 +112,6 @@ def submit_module_build_from_yaml( ): yaml_file = to_text_type(handle.read()) mmd = load_mmd(yaml_file) - dt = datetime.utcfromtimestamp(int(time.time())) if hasattr(handle, "filename"): def_name = str(os.path.splitext(os.path.basename(handle.filename))[0]) elif not mmd.get_module_name(): @@ -120,19 +119,20 @@ def submit_module_build_from_yaml( "The module's name was not present in the modulemd file. Please use the " '"module_name" parameter' ) - def_version = int(dt.strftime("%Y%m%d%H%M%S")) module_name = mmd.get_module_name() or def_name module_stream = stream or mmd.get_stream_name() or "master" if module_name != mmd.get_module_name() or module_stream != mmd.get_stream_name(): # This is how you set the name and stream in the modulemd mmd = mmd.copy(module_name, module_stream) - mmd.set_version(mmd.get_version() or def_version) if skiptests: buildopts = mmd.get_buildopts() or Modulemd.Buildopts() macros = buildopts.get_rpm_macros() or "" buildopts.set_rpm_macros(macros + "\n\n%__spec_check_pre exit 0\n") mmd.set_buildopts(buildopts) - return submit_module_build(db_session, username, mmd, params) + + module_stream_version = provide_module_stream_version_from_mmd(mmd) + + return submit_module_build(db_session, username, mmd, params, module_stream_version) _url_check_re = re.compile(r"^[^:/]+:.*$") @@ -148,7 +148,9 @@ def submit_module_build_from_scm(db_session, username, params, allow_local_url=F url = "file://" + url mmd, scm = fetch_mmd(url, branch, allow_local_url) - return submit_module_build(db_session, username, mmd, params) + module_stream_version = int(scm.version) + + return submit_module_build(db_session, username, mmd, params, module_stream_version) def _apply_dep_overrides(mmd, params): @@ -521,7 +523,7 @@ def _process_support_streams(db_session, mmd, params): _modify_buildtime_streams(db_session, mmd, new_streams_func) -def submit_module_build(db_session, username, mmd, params): +def submit_module_build(db_session, username, mmd, params, module_stream_version): """ Submits new module build. @@ -532,13 +534,6 @@ def submit_module_build(db_session, username, mmd, params): :rtype: list with ModuleBuild :return: List with submitted module builds. """ - log.debug( - "Submitted %s module build for %s:%s:%s", - ("scratch" if params.get("scratch", False) else "normal"), - mmd.get_module_name(), - mmd.get_stream_name(), - mmd.get_version(), - ) raise_if_stream_ambigous = False default_streams = {} @@ -550,11 +545,18 @@ def submit_module_build(db_session, username, mmd, params): if "default_streams" in params: default_streams = params["default_streams"] - xmd = mmd.get_xmd() - # we check if static contexts are enabled by the `contexts` property defined by the user i - # as an build option. - static_context = "mbs_options" in xmd and "contexts" in xmd["mbs_options"] - input_mmds = generate_mmds_from_static_contexts(mmd) if static_context else [mmd] + input_mmds, static_context = process_module_context_configuration(mmd) + + for mmd in input_mmds: + mmd.set_version(module_stream_version) + + log.debug( + "Submitted %s module build for %s:%s:%s", + ("scratch" if params.get("scratch", False) else "normal"), + input_mmds[0].get_module_name(), + input_mmds[0].get_stream_name(), + input_mmds[0].get_version(), + ) mmds = [] for mmd in input_mmds: @@ -690,3 +692,62 @@ def submit_module_build(db_session, username, mmd, params): raise Conflict(err_msg) return modules + + +def process_module_context_configuration(mmd): + """ + Processes initial module metadata context configurations and creates individual module + metadata for each context, if static context configuration is present. + + :param Modulemd.ModuleStream packager_mmd: Packager (initial) modulemd which kickstarts + the build. + :rtype: list with ModuleBuild + :return: list of generated module metadata from context configurations. + """ + mdversion = mmd.get_mdversion() + static_context = False + + # we check what version of the metadata format we are using. + if mdversion == 3: + # v3 we always talking about a new build and the static context + # will be always True + static_context = True + mdindex = mmd.convert_to_index() + streams = mdindex.search_streams() + + for stream in streams: + if not stream.is_static_context(): + stream.set_static_context() + + # we get the dependenices of the stream + deps = stream.get_dependencies() + # with v3 packager format the output v2 stream will always have + # only one set of dependecies. We need to remove the platform + # virtual module from runtime dependencies as it is not desired. + modules = deps[0].get_runtime_modules() + module_streams = [(m, deps[0].get_runtime_streams(m)[0]) for m in modules + if m not in conf.base_module_names] + deps[0].clear_runtime_dependencies() + + for module_stream in module_streams: + module, stream = module_stream + deps[0].add_runtime_stream(module, stream) + + return streams, static_context + else: + xmd = mmd.get_xmd() + # check if we are handling rebuild of a static context module + if "mbs" in xmd: + # check if it is a static context + if "static_context" in xmd["mbs"] or mmd.is_static_context(): + static_context = True + return [mmd], static_context + + # we check if static contexts are enabled by the `contexts` property defined by the user i + # as an build option. + static_context = "mbs_options" in xmd and "contexts" in xmd["mbs_options"] + # if the static context configuration exists we expand it. If not we just return + # the mmd unchanged, for futher processing. + streams = generate_mmds_from_static_contexts(mmd) if static_context else [mmd] + + return streams, static_context diff --git a/tests/staged_data/static_context_v2.yaml b/tests/staged_data/static_context_v2.yaml new file mode 100644 index 00000000..d854c488 --- /dev/null +++ b/tests/staged_data/static_context_v2.yaml @@ -0,0 +1,32 @@ +document: modulemd +version: 2 +data: + name: app + stream: test + summary: "A test module" + description: > + "A test module stream" + license: + module: [ MIT ] + dependencies: + - buildrequires: + platform: [] + gtk: [] + requires: + platform: [] + gtk: [] + xmd: + mbs_options: + contexts: + context1: + buildrequires: + platform: f28 + requires: + platform: f28 + gtk: 1 + context2: + buildrequires: + platform: f28 + requires: + platform: f28 + gtk: 2 \ No newline at end of file diff --git a/tests/staged_data/v3/mmd_packager.yaml b/tests/staged_data/v3/mmd_packager.yaml new file mode 100644 index 00000000..693e1e30 --- /dev/null +++ b/tests/staged_data/v3/mmd_packager.yaml @@ -0,0 +1,41 @@ +document: modulemd-packager + +# Module metadata format version +version: 3 +data: + name: foo + stream: latest + summary: An example module + description: >- + A module for the demonstration of the metadata format. Also, + the obligatory lorem ipsum dolor sit amet goes right here. + license: + - MIT + xmd: + some_key: some_data + configurations: + - context: CTX1 + platform: f28 + requires: + nginx: [1] + - context: CTX2 + platform: f29 + references: + community: http://www.example.com/ + documentation: http://www.example.com/ + tracker: http://www.example.com/ + profiles: + container: + rpms: + - foo + api: + rpms: + - foo + components: + rpms: + foo: + name: foo + rationale: We need this to demonstrate stuff. + repository: https://pagure.io/foo.git + cache: https://example.com/cache + ref: 26ca0c0 \ No newline at end of file diff --git a/tests/test_builder/test_content_generator.py b/tests/test_builder/test_content_generator.py index 54080dd5..5dee0d7a 100644 --- a/tests/test_builder/test_content_generator.py +++ b/tests/test_builder/test_content_generator.py @@ -203,10 +203,10 @@ class TestBuild: assert len(mmd.read()) == 1271 with io.open(path.join(dir_path, "modulemd.x86_64.txt"), encoding="utf-8") as mmd: - assert len(mmd.read()) == 349 + assert len(mmd.read()) == 351 with io.open(path.join(dir_path, "modulemd.i686.txt"), encoding="utf-8") as mmd: - assert len(mmd.read()) == 347 + assert len(mmd.read()) == 349 @patch("koji.ClientSession") def test_tag_cg_build(self, ClientSession): @@ -318,7 +318,7 @@ class TestBuild: assert ret == { "arch": "x86_64", "buildroot_id": 1, - "checksum": "580e1f880e0872bed58591c55eeb6e7e", + "checksum": "3b4b748bd14ae440176a6ea1fe649218", "checksum_type": "md5", "components": [ { @@ -333,7 +333,7 @@ class TestBuild: ], "extra": {"typeinfo": {"module": {}}}, "filename": "modulemd.x86_64.txt", - "filesize": 409, + "filesize": 411, "type": "file", } diff --git a/tests/test_common/test_utils.py b/tests/test_common/test_utils.py index 6d1602a2..6e639156 100644 --- a/tests/test_common/test_utils.py +++ b/tests/test_common/test_utils.py @@ -1,14 +1,17 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT from __future__ import absolute_import +import mock import pytest from module_build_service.common import models from module_build_service.common.errors import UnprocessableEntity -from module_build_service.common.utils import import_mmd, load_mmd +from module_build_service.common.utils import (import_mmd, load_mmd, + provide_module_stream_version_from_mmd, + provide_module_stream_version_from_timestamp) from module_build_service.scheduler.db_session import db_session -from tests import read_staged_data +from tests import read_staged_data, staged_data_filename @pytest.mark.parametrize("context", ["c1", None]) @@ -151,3 +154,136 @@ def test_import_mmd_dont_remove_dropped_virtual_streams_associated_with_other_mo # The overlapped f30 should be still there. db_session.refresh(another_module_build) assert ["f29", "f30"] == sorted(item.name for item in another_module_build.virtual_streams) + + +def test_load_mmd_v2(): + """ + Test to check if we can load a v2 packager file. + """ + mmd = load_mmd(read_staged_data("testmodule_v2.yaml")) + + assert mmd.get_mdversion() == 2 + + dep = mmd.get_dependencies() + btm = dep[0].get_buildtime_modules() + rtm = dep[0].get_runtime_modules() + + assert len(btm) == 1 + assert len(rtm) == 1 + assert btm[0] == "platform" + assert rtm[0] == "platform" + + bts = dep[0].get_buildtime_streams(btm[0]) + rts = dep[0].get_runtime_streams(rtm[0]) + + assert len(bts) == 1 + assert len(rts) == 1 + assert bts[0] == "f28" + assert rts[0] == "f28" + + +def test_load_mmd_v3(): + """ + Test to check if we can load a v3 packager file. + """ + mmd = load_mmd(read_staged_data("v3/mmd_packager.yaml")) + + assert mmd.get_mdversion() == 3 + contexts = mmd.get_build_config_contexts_as_strv() + bc1 = mmd.get_build_config(contexts[0]) + assert bc1.get_context() == "CTX1" + assert bc1.get_platform() == "f28" + btm1 = bc1.get_buildtime_modules_as_strv() + rtm1 = bc1.get_runtime_modules_as_strv() + assert len(btm1) == 0 + assert len(rtm1) == 1 + assert rtm1[0] == "nginx" + assert bc1.get_runtime_requirement_stream(rtm1[0]) == "1" + + bc2 = mmd.get_build_config(contexts[1]) + assert bc2.get_context() == "CTX2" + assert bc2.get_platform() == "f29" + assert not bc2.get_buildtime_modules_as_strv() + assert not bc2.get_runtime_modules_as_strv() + + +def test_provide_module_stream_version_from_timestamp(): + ux_timestamp = "1613048427" + version = provide_module_stream_version_from_timestamp(ux_timestamp) + assert version == 20210211130027 + + +@mock.patch("module_build_service.common.utils.time") +def test_provide_module_stream_version_from_timestamp_no_params(mock_time): + mock_time.time.return_value = "1613048427" + + version = provide_module_stream_version_from_timestamp() + + assert version == 20210211130027 + + +def test_provide_module_stream_version_from_mmd_v2(): + expected_version = 20210211130027 + + mmd = load_mmd(read_staged_data("testmodule_v2.yaml")) + mmd.set_version(expected_version) + + version = provide_module_stream_version_from_mmd(mmd) + + assert version == expected_version + + +@mock.patch("module_build_service.common.utils.time") +def test_provide_module_stream_version_from_mmd_v2_no_set_version(mock_time): + mock_time.time.return_value = "1613048427" + + mmd = load_mmd(read_staged_data("testmodule_v2.yaml")) + + version = provide_module_stream_version_from_mmd(mmd) + + assert version == 20210211130027 + + +@mock.patch("module_build_service.common.utils.time") +def test_provide_module_stream_version_from_mmd_v3(mock_time): + mock_time.time.return_value = "1613048427" + + mmd = load_mmd(read_staged_data("v3/mmd_packager.yaml")) + + version = provide_module_stream_version_from_mmd(mmd) + + assert version == 20210211130027 + + +def test_load_mmd_bad_mdversion_raise(): + bad_mdversion_mmd = read_staged_data("v3/mmd_packager.yaml").replace("3", "4") + + with pytest.raises(UnprocessableEntity) as e: + load_mmd(bad_mdversion_mmd) + + err_msg = e.value.args[0] + assert "modulemd is invalid" in err_msg + assert "Invalid mdversion" in err_msg + assert "modulemd-yaml-error-quark" in err_msg + + +def test_load_mmd_missing_file_raise(): + bad_path = "../something/something" + + with pytest.raises(UnprocessableEntity) as e: + load_mmd(bad_path, is_file=True) + + err_msg = e.value.args[0] + assert "file ../something/something was not found" in err_msg + + +def test_load_mmd_file_wrong_data_raise(): + bad_file = staged_data_filename("bad.yaml") + + with pytest.raises(UnprocessableEntity) as e: + load_mmd(bad_file, is_file=True) + + err_msg = e.value.args[0] + assert "modulemd-yaml-error-quark" in err_msg + assert "Parse error identifying document type and version" in err_msg + assert "The modulemd bad.yaml is invalid" in err_msg diff --git a/tests/test_web/test_submit.py b/tests/test_web/test_submit.py index 415c67cc..8a75bf53 100644 --- a/tests/test_web/test_submit.py +++ b/tests/test_web/test_submit.py @@ -12,15 +12,19 @@ from werkzeug.datastructures import FileStorage from module_build_service.common import models from module_build_service.common.errors import ValidationError -from module_build_service.common.utils import mmd_to_str, load_mmd +from module_build_service.common.utils import (mmd_to_str, load_mmd, + provide_module_stream_version_from_timestamp) from module_build_service.scheduler.db_session import db_session from module_build_service.web.submit import ( - get_prefixed_version, submit_module_build, submit_module_build_from_yaml + get_prefixed_version, submit_module_build, submit_module_build_from_yaml, + process_module_context_configuration ) from tests import ( scheduler_init_data, make_module_in_db, make_module, + read_staged_data, + init_data, ) @@ -50,43 +54,12 @@ class TestSubmit: is set to False. """ - yaml_str = """ -document: modulemd -version: 2 -data: - name: app - stream: test - summary: "A test module" - description: > - "A test module stream" - license: - module: [ MIT ] - dependencies: - - buildrequires: - platform: [] - gtk: [] - requires: - platform: [] - gtk: [] - xmd: - mbs_options: - contexts: - context1: - buildrequires: - platform: f28 - requires: - platform: f28 - gtk: 1 - context2: - buildrequires: - platform: f28 - requires: - platform: f28 - gtk: 2 - """ + yaml_str = read_staged_data("static_context_v2") mmd = load_mmd(yaml_str) - builds = submit_module_build(db_session, "app", mmd, {}) + ux_timestamp = "1613048427" + version = provide_module_stream_version_from_timestamp(ux_timestamp) + builds = submit_module_build(db_session, "app", mmd, {}, version) expected_context = ["context1", "context2"] @@ -96,6 +69,7 @@ data: assert build.context in expected_context mmd = build.mmd() xmd = mmd.get_xmd() + assert mmd.get_version() == int("28" + str(version)) assert "mbs_options" not in xmd assert xmd["mbs"]["static_context"] @@ -105,38 +79,19 @@ data: are more options configured then `contexts` option.. """ - yaml_str = """ -document: modulemd -version: 2 -data: - name: app - stream: test1 - summary: "A test module" - description: > - "A test module stream" - license: - module: [ MIT ] - dependencies: - - buildrequires: - platform: [] - gtk: [] - requires: - platform: [] - gtk: [] - xmd: - mbs_options: - contexts: - context1: - buildrequires: - platform: f28 - requires: - platform: f28 - gtk: 1 - another_option: "test" - """ + yaml_str = read_staged_data("static_context_v2") mmd = load_mmd(yaml_str) + xmd = mmd.get_xmd() + xmd["mbs_options"]["another_option"] = "test" + xmd["mbs_options"]["contexts"] = { + "context3": xmd["mbs_options"]["contexts"]["context1"] + } + mmd.set_xmd(xmd) - builds = submit_module_build(db_session, "app", mmd, {}) + ux_timestamp = "1613048427" + version = provide_module_stream_version_from_timestamp(ux_timestamp) + + builds = submit_module_build(db_session, "app", mmd, {}, version) assert len(builds) == 1 mmd = builds[0].mmd() @@ -145,6 +100,123 @@ data: assert "another_option" in xmd["mbs_options"] assert "test" == xmd["mbs_options"]["another_option"] + def test_submit_build_module_context_configurations(self): + """ + With the introduction of the v3 packager yaml format we replace MSE with static contexts. + This test tests the submission of such a packager file. + """ + init_data(multiple_stream_versions=True) + yaml_str = read_staged_data("v3/mmd_packager") + mmd = load_mmd(yaml_str) + ux_timestamp = "1613048427" + version = provide_module_stream_version_from_timestamp(ux_timestamp) + builds = submit_module_build(db_session, "foo", mmd, {}, version) + + assert len(builds) == 2 + + expected_deps = {"CTX1": {"buildrequires": {"platform": ["f28"]}, + "requires": {"nginx": ["1"]}}, + "CTX2": {"buildrequires": {"platform": ["f29.2.0"]}, + "requires": {}}} + + for build in builds: + mmd = build.mmd() + context = mmd.get_context() + assert context in expected_deps + assert mmd.get_mdversion() == 2 + + deps = mmd.get_dependencies()[0] + + btms = deps.get_buildtime_modules() + rtms = deps.get_runtime_modules() + + for btm in btms: + expected_stream = expected_deps[context]["buildrequires"][btm][0] + actual_stream = deps.get_buildtime_streams(btm)[0] + assert expected_stream == actual_stream + + for rtm in rtms: + expected_stream = expected_deps[context]["requires"][rtm][0] + actual_stream = deps.get_runtime_streams(rtm)[0] + assert expected_stream == actual_stream + + xmd = mmd.get_xmd() + assert "mbs_options" not in xmd + assert xmd["mbs"]["static_context"] + + +class TestProcessModuleContextConfiguration: + """ + With the introduction of static contexts, we have now several different configuration of + static contexts for different major/minor RHEL releases. + """ + + def test_process_mse_configuration(self): + """ + Testing the processing of MSE type packager (v2) file i. e. no static context. + """ + + yaml_str = read_staged_data("static_context_v2") + mmd = load_mmd(yaml_str) + mmd.set_xmd({}) + + streams, static_context = process_module_context_configuration(mmd) + + assert not static_context + assert len(streams) == 1 + assert not mmd.get_context() + + def test_process_initial_xmd_static_configuration(self): + """ + Testing the processing of MSE type packager (v2) file with static context + configuration in XMD + """ + yaml_str = read_staged_data("static_context_v2") + mmd = load_mmd(yaml_str) + + streams, static_context = process_module_context_configuration(mmd) + + assert static_context + assert len(streams) == 2 + expected_contexts = ["context1", "context2"] + for stream in streams: + assert stream.get_context() in expected_contexts + + def test_process_xmd_static_context_rebuild(self): + """ + Testing the processing of MSE type file (v2) used for rebuild with set static_context in + mbs metadata in xmd. + """ + + yaml_str = read_staged_data("testmodule_v2") + mmd = load_mmd(yaml_str) + mmd.set_context("context1") + mmd.set_xmd({"mbs": {"static_context": True}}) + + streams, static_context = process_module_context_configuration(mmd) + + assert static_context + assert len(streams) == 1 + assert streams[0].get_context() == "context1" + + def test_process_v3_packager_file(self): + """ + Testing the processing of v3 packager file with static context configurations. + """ + + yaml_str = read_staged_data("v3/mmd_packager") + mmd = load_mmd(yaml_str) + + streams, static_context = process_module_context_configuration(mmd) + + assert static_context + assert len(streams) == 2 + expected_contexts = ["CTX1", "CTX2"] + for stream in streams: + assert stream.get_mdversion() == 2 + assert stream.get_context() in expected_contexts + assert stream.is_static_context() + @pytest.mark.usefixtures("reuse_component_init_data") class TestUtilsComponentReuse: @@ -200,10 +272,13 @@ class TestUtilsComponentReuse: mmd1_copy = mmd1.copy() mmd1_copy.set_xmd({}) - builds = submit_module_build(db_session, "foo", mmd1_copy, {}) - ret = {b.mmd().get_context(): b.state for b in builds} - assert ret == {"c1": models.BUILD_STATES["ready"], "c2": models.BUILD_STATES["init"]} + ux_timestamp = "1613048427" + version = provide_module_stream_version_from_timestamp(ux_timestamp) + builds = submit_module_build(db_session, "foo", mmd1_copy, {}, version) + ret = {b.mmd().get_context(): b.state for b in builds} + + assert ret == {"c1": models.BUILD_STATES["ready"], "c2": models.BUILD_STATES["init"]} assert builds[0].siblings(db_session) == [builds[1].id] assert builds[1].siblings(db_session) == [builds[0].id] @@ -223,10 +298,13 @@ class TestUtilsComponentReuse: mmd_copy = mmd.copy() mmd_copy.set_xmd({}) + ux_timestamp = "1613048427" + version = provide_module_stream_version_from_timestamp(ux_timestamp) + with pytest.raises( ValidationError, match="Only scratch module builds can be built from this branch.", ): - submit_module_build(db_session, "foo", mmd_copy, {"branch": "private-foo"}) + submit_module_build(db_session, "foo", mmd_copy, {"branch": "private-foo"}, version) - submit_module_build(db_session, "foo", mmd_copy, {"branch": "otherbranch"}) + submit_module_build(db_session, "foo", mmd_copy, {"branch": "otherbranch"}, version) diff --git a/tests/test_web/test_views.py b/tests/test_web/test_views.py index d520b4e7..31a82ce1 100644 --- a/tests/test_web/test_views.py +++ b/tests/test_web/test_views.py @@ -1259,9 +1259,10 @@ class TestSubmitBuild: ) data = json.loads(rv.data) assert re.match( - r"The modulemd .* is invalid\. Please verify the syntax is correct", + r"The modulemd .* is invalid\. ", data["message"] ) + assert "Please verify the syntax is correct" in data["message"] assert data["status"] == 422 assert data["error"] == "Unprocessable Entity" @@ -2796,8 +2797,9 @@ class TestImportBuild: assert data["error"] == "Unprocessable Entity" assert re.match( - r"The modulemd .* is invalid\. Please verify the syntax is correct", data["message"] + r"The modulemd .* is invalid\. ", data["message"] ) + assert "Please verify the syntax is correct" in data["message"] @pytest.mark.usefixtures("provide_test_client")