modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-23 10:12:03 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added the support for the new libmodulemd v3 packager format.

Browse Source 
This will enable building modules from the new libmodulemd v3 format.
This format fixes major issue with modularity which enables clear
upgrade paths for multicontext modules.

Signed-off-by: Martin Curlej <mcurlej@redhat.com>
This commit is contained in:
Martin Curlej
2021-01-06 14:53:11 +01:00
parent a2bd848aea
commit 7976b1f084
10 changed files with 512 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
tests/staged_data/static_context_v2.yaml Normal file
View File

@@ -0,0 +1,32 @@
document: modulemd
version: 2
data:
name: app
stream: test
summary: "A test module"
description: >
"A test module stream"
license:
module: [ MIT ]
dependencies:
- buildrequires:
platform: []
gtk: []
requires:
platform: []
gtk: []
xmd:
mbs_options:
contexts:
context1:
buildrequires:
platform: f28
requires:
platform: f28
gtk: 1
context2:
buildrequires:
platform: f28
requires:
platform: f28
gtk: 2

41
tests/staged_data/v3/mmd_packager.yaml Normal file
View File

@@ -0,0 +1,41 @@
document: modulemd-packager
# Module metadata format version
version: 3
data:
name: foo
stream: latest
summary: An example module
description: >-
A module for the demonstration of the metadata format. Also,
the obligatory lorem ipsum dolor sit amet goes right here.
license:
- MIT
xmd:
some_key: some_data
configurations:
- context: CTX1
platform: f28
requires:
nginx: [1]
- context: CTX2
platform: f29
references:
community: http://www.example.com/
documentation: http://www.example.com/
tracker: http://www.example.com/
profiles:
container:
rpms:
- foo
api:
rpms:
- foo
components:
rpms:
foo:
name: foo
rationale: We need this to demonstrate stuff.
repository: https://pagure.io/foo.git
cache: https://example.com/cache
ref: 26ca0c0

8
tests/test_builder/test_content_generator.py
View File

@@ -203,10 +203,10 @@ class TestBuild:
assert len(mmd.read()) == 1271
with io.open(path.join(dir_path, "modulemd.x86_64.txt"), encoding="utf-8") as mmd:
assert len(mmd.read()) == 349
assert len(mmd.read()) == 351
with io.open(path.join(dir_path, "modulemd.i686.txt"), encoding="utf-8") as mmd:
assert len(mmd.read()) == 347
assert len(mmd.read()) == 349
@patch("koji.ClientSession")
def test_tag_cg_build(self, ClientSession):
@@ -318,7 +318,7 @@ class TestBuild:
assert ret == {
"arch": "x86_64",
"buildroot_id": 1,
"checksum": "580e1f880e0872bed58591c55eeb6e7e",
"checksum": "3b4b748bd14ae440176a6ea1fe649218",
"checksum_type": "md5",
"components": [
{
@@ -333,7 +333,7 @@ class TestBuild:
],
"extra": {"typeinfo": {"module": {}}},
"filename": "modulemd.x86_64.txt",
"filesize": 409,
"filesize": 411,
"type": "file",
}

140
tests/test_common/test_utils.py
View File

@@ -1,14 +1,17 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
import mock
import pytest
from module_build_service.common import models
from module_build_service.common.errors import UnprocessableEntity
from module_build_service.common.utils import import_mmd, load_mmd
from module_build_service.common.utils import (import_mmd, load_mmd,
provide_module_stream_version_from_mmd,
provide_module_stream_version_from_timestamp)
from module_build_service.scheduler.db_session import db_session
from tests import read_staged_data
from tests import read_staged_data, staged_data_filename
@pytest.mark.parametrize("context", ["c1", None])
@@ -151,3 +154,136 @@ def test_import_mmd_dont_remove_dropped_virtual_streams_associated_with_other_mo
# The overlapped f30 should be still there.
db_session.refresh(another_module_build)
assert ["f29", "f30"] == sorted(item.name for item in another_module_build.virtual_streams)
def test_load_mmd_v2():
"""
Test to check if we can load a v2 packager file.
"""
mmd = load_mmd(read_staged_data("testmodule_v2.yaml"))
assert mmd.get_mdversion() == 2
dep = mmd.get_dependencies()
btm = dep[0].get_buildtime_modules()
rtm = dep[0].get_runtime_modules()
assert len(btm) == 1
assert len(rtm) == 1
assert btm[0] == "platform"
assert rtm[0] == "platform"
bts = dep[0].get_buildtime_streams(btm[0])
rts = dep[0].get_runtime_streams(rtm[0])
assert len(bts) == 1
assert len(rts) == 1
assert bts[0] == "f28"
assert rts[0] == "f28"
def test_load_mmd_v3():
"""
Test to check if we can load a v3 packager file.
"""
mmd = load_mmd(read_staged_data("v3/mmd_packager.yaml"))
assert mmd.get_mdversion() == 3
contexts = mmd.get_build_config_contexts_as_strv()
bc1 = mmd.get_build_config(contexts[0])
assert bc1.get_context() == "CTX1"
assert bc1.get_platform() == "f28"
btm1 = bc1.get_buildtime_modules_as_strv()
rtm1 = bc1.get_runtime_modules_as_strv()
assert len(btm1) == 0
assert len(rtm1) == 1
assert rtm1[0] == "nginx"
assert bc1.get_runtime_requirement_stream(rtm1[0]) == "1"
bc2 = mmd.get_build_config(contexts[1])
assert bc2.get_context() == "CTX2"
assert bc2.get_platform() == "f29"
assert not bc2.get_buildtime_modules_as_strv()
assert not bc2.get_runtime_modules_as_strv()
def test_provide_module_stream_version_from_timestamp():
ux_timestamp = "1613048427"
version = provide_module_stream_version_from_timestamp(ux_timestamp)
assert version == 20210211130027
@mock.patch("module_build_service.common.utils.time")
def test_provide_module_stream_version_from_timestamp_no_params(mock_time):
mock_time.time.return_value = "1613048427"
version = provide_module_stream_version_from_timestamp()
assert version == 20210211130027
def test_provide_module_stream_version_from_mmd_v2():
expected_version = 20210211130027
mmd = load_mmd(read_staged_data("testmodule_v2.yaml"))
mmd.set_version(expected_version)
version = provide_module_stream_version_from_mmd(mmd)
assert version == expected_version
@mock.patch("module_build_service.common.utils.time")
def test_provide_module_stream_version_from_mmd_v2_no_set_version(mock_time):
mock_time.time.return_value = "1613048427"
mmd = load_mmd(read_staged_data("testmodule_v2.yaml"))
version = provide_module_stream_version_from_mmd(mmd)
assert version == 20210211130027
@mock.patch("module_build_service.common.utils.time")
def test_provide_module_stream_version_from_mmd_v3(mock_time):
mock_time.time.return_value = "1613048427"
mmd = load_mmd(read_staged_data("v3/mmd_packager.yaml"))
version = provide_module_stream_version_from_mmd(mmd)
assert version == 20210211130027
def test_load_mmd_bad_mdversion_raise():
bad_mdversion_mmd = read_staged_data("v3/mmd_packager.yaml").replace("3", "4")
with pytest.raises(UnprocessableEntity) as e:
load_mmd(bad_mdversion_mmd)
err_msg = e.value.args[0]
assert "modulemd is invalid" in err_msg
assert "Invalid mdversion" in err_msg
assert "modulemd-yaml-error-quark" in err_msg
def test_load_mmd_missing_file_raise():
bad_path = "../something/something"
with pytest.raises(UnprocessableEntity) as e:
load_mmd(bad_path, is_file=True)
err_msg = e.value.args[0]
assert "file ../something/something was not found" in err_msg
def test_load_mmd_file_wrong_data_raise():
bad_file = staged_data_filename("bad.yaml")
with pytest.raises(UnprocessableEntity) as e:
load_mmd(bad_file, is_file=True)
err_msg = e.value.args[0]
assert "modulemd-yaml-error-quark" in err_msg
assert "Parse error identifying document type and version" in err_msg
assert "The modulemd bad.yaml is invalid" in err_msg

222
tests/test_web/test_submit.py
View File

@@ -12,15 +12,19 @@ from werkzeug.datastructures import FileStorage
from module_build_service.common import models
from module_build_service.common.errors import ValidationError
from module_build_service.common.utils import mmd_to_str, load_mmd
from module_build_service.common.utils import (mmd_to_str, load_mmd,
provide_module_stream_version_from_timestamp)
from module_build_service.scheduler.db_session import db_session
from module_build_service.web.submit import (
get_prefixed_version, submit_module_build, submit_module_build_from_yaml
get_prefixed_version, submit_module_build, submit_module_build_from_yaml,
process_module_context_configuration
)
from tests import (
scheduler_init_data,
make_module_in_db,
make_module,
read_staged_data,
init_data,
)
@@ -50,43 +54,12 @@ class TestSubmit:
is set to False.
"""
yaml_str = """
document: modulemd
version: 2
data:
name: app
stream: test
summary: "A test module"
description: >
"A test module stream"
license:
module: [ MIT ]
dependencies:
- buildrequires:
platform: []
gtk: []
requires:
platform: []
gtk: []
xmd:
mbs_options:
contexts:
context1:
buildrequires:
platform: f28
requires:
platform: f28
gtk: 1
context2:
buildrequires:
platform: f28
requires:
platform: f28
gtk: 2
"""
yaml_str = read_staged_data("static_context_v2")
mmd = load_mmd(yaml_str)
builds = submit_module_build(db_session, "app", mmd, {})
ux_timestamp = "1613048427"
version = provide_module_stream_version_from_timestamp(ux_timestamp)
builds = submit_module_build(db_session, "app", mmd, {}, version)
expected_context = ["context1", "context2"]
@@ -96,6 +69,7 @@ data:
assert build.context in expected_context
mmd = build.mmd()
xmd = mmd.get_xmd()
assert mmd.get_version() == int("28" + str(version))
assert "mbs_options" not in xmd
assert xmd["mbs"]["static_context"]
@@ -105,38 +79,19 @@ data:
are more options configured then `contexts` option..
"""
yaml_str = """
document: modulemd
version: 2
data:
name: app
stream: test1
summary: "A test module"
description: >
"A test module stream"
license:
module: [ MIT ]
dependencies:
- buildrequires:
platform: []
gtk: []
requires:
platform: []
gtk: []
xmd:
mbs_options:
contexts:
context1:
buildrequires:
platform: f28
requires:
platform: f28
gtk: 1
another_option: "test"
"""
yaml_str = read_staged_data("static_context_v2")
mmd = load_mmd(yaml_str)
xmd = mmd.get_xmd()
xmd["mbs_options"]["another_option"] = "test"
xmd["mbs_options"]["contexts"] = {
"context3": xmd["mbs_options"]["contexts"]["context1"]
}
mmd.set_xmd(xmd)
builds = submit_module_build(db_session, "app", mmd, {})
ux_timestamp = "1613048427"
version = provide_module_stream_version_from_timestamp(ux_timestamp)
builds = submit_module_build(db_session, "app", mmd, {}, version)
assert len(builds) == 1
mmd = builds[0].mmd()
@@ -145,6 +100,123 @@ data:
assert "another_option" in xmd["mbs_options"]
assert "test" == xmd["mbs_options"]["another_option"]
def test_submit_build_module_context_configurations(self):
"""
With the introduction of the v3 packager yaml format we replace MSE with static contexts.
This test tests the submission of such a packager file.
"""
init_data(multiple_stream_versions=True)
yaml_str = read_staged_data("v3/mmd_packager")
mmd = load_mmd(yaml_str)
ux_timestamp = "1613048427"
version = provide_module_stream_version_from_timestamp(ux_timestamp)
builds = submit_module_build(db_session, "foo", mmd, {}, version)
assert len(builds) == 2
expected_deps = {"CTX1": {"buildrequires": {"platform": ["f28"]},
"requires": {"nginx": ["1"]}},
"CTX2": {"buildrequires": {"platform": ["f29.2.0"]},
"requires": {}}}
for build in builds:
mmd = build.mmd()
context = mmd.get_context()
assert context in expected_deps
assert mmd.get_mdversion() == 2
deps = mmd.get_dependencies()[0]
btms = deps.get_buildtime_modules()
rtms = deps.get_runtime_modules()
for btm in btms:
expected_stream = expected_deps[context]["buildrequires"][btm][0]
actual_stream = deps.get_buildtime_streams(btm)[0]
assert expected_stream == actual_stream
for rtm in rtms:
expected_stream = expected_deps[context]["requires"][rtm][0]
actual_stream = deps.get_runtime_streams(rtm)[0]
assert expected_stream == actual_stream
xmd = mmd.get_xmd()
assert "mbs_options" not in xmd
assert xmd["mbs"]["static_context"]
class TestProcessModuleContextConfiguration:
"""
With the introduction of static contexts, we have now several different configuration of
static contexts for different major/minor RHEL releases.
"""
def test_process_mse_configuration(self):
"""
Testing the processing of MSE type packager (v2) file i. e. no static context.
"""
yaml_str = read_staged_data("static_context_v2")
mmd = load_mmd(yaml_str)
mmd.set_xmd({})
streams, static_context = process_module_context_configuration(mmd)
assert not static_context
assert len(streams) == 1
assert not mmd.get_context()
def test_process_initial_xmd_static_configuration(self):
"""
Testing the processing of MSE type packager (v2) file with static context
configuration in XMD
"""
yaml_str = read_staged_data("static_context_v2")
mmd = load_mmd(yaml_str)
streams, static_context = process_module_context_configuration(mmd)
assert static_context
assert len(streams) == 2
expected_contexts = ["context1", "context2"]
for stream in streams:
assert stream.get_context() in expected_contexts
def test_process_xmd_static_context_rebuild(self):
"""
Testing the processing of MSE type file (v2) used for rebuild with set static_context in
mbs metadata in xmd.
"""
yaml_str = read_staged_data("testmodule_v2")
mmd = load_mmd(yaml_str)
mmd.set_context("context1")
mmd.set_xmd({"mbs": {"static_context": True}})
streams, static_context = process_module_context_configuration(mmd)
assert static_context
assert len(streams) == 1
assert streams[0].get_context() == "context1"
def test_process_v3_packager_file(self):
"""
Testing the processing of v3 packager file with static context configurations.
"""
yaml_str = read_staged_data("v3/mmd_packager")
mmd = load_mmd(yaml_str)
streams, static_context = process_module_context_configuration(mmd)
assert static_context
assert len(streams) == 2
expected_contexts = ["CTX1", "CTX2"]
for stream in streams:
assert stream.get_mdversion() == 2
assert stream.get_context() in expected_contexts
assert stream.is_static_context()
@pytest.mark.usefixtures("reuse_component_init_data")
class TestUtilsComponentReuse:
@@ -200,10 +272,13 @@ class TestUtilsComponentReuse:
mmd1_copy = mmd1.copy()
mmd1_copy.set_xmd({})
builds = submit_module_build(db_session, "foo", mmd1_copy, {})
ret = {b.mmd().get_context(): b.state for b in builds}
assert ret == {"c1": models.BUILD_STATES["ready"], "c2": models.BUILD_STATES["init"]}
ux_timestamp = "1613048427"
version = provide_module_stream_version_from_timestamp(ux_timestamp)
builds = submit_module_build(db_session, "foo", mmd1_copy, {}, version)
ret = {b.mmd().get_context(): b.state for b in builds}
assert ret == {"c1": models.BUILD_STATES["ready"], "c2": models.BUILD_STATES["init"]}
assert builds[0].siblings(db_session) == [builds[1].id]
assert builds[1].siblings(db_session) == [builds[0].id]
@@ -223,10 +298,13 @@ class TestUtilsComponentReuse:
mmd_copy = mmd.copy()
mmd_copy.set_xmd({})
ux_timestamp = "1613048427"
version = provide_module_stream_version_from_timestamp(ux_timestamp)
with pytest.raises(
ValidationError,
match="Only scratch module builds can be built from this branch.",
):
submit_module_build(db_session, "foo", mmd_copy, {"branch": "private-foo"})
submit_module_build(db_session, "foo", mmd_copy, {"branch": "private-foo"}, version)
submit_module_build(db_session, "foo", mmd_copy, {"branch": "otherbranch"})
submit_module_build(db_session, "foo", mmd_copy, {"branch": "otherbranch"}, version)

6
tests/test_web/test_views.py
View File

@@ -1259,9 +1259,10 @@ class TestSubmitBuild:
)
data = json.loads(rv.data)
assert re.match(
r"The modulemd .* is invalid\. Please verify the syntax is correct",
r"The modulemd .* is invalid\. ",
data["message"]
)
assert "Please verify the syntax is correct" in data["message"]
assert data["status"] == 422
assert data["error"] == "Unprocessable Entity"
@@ -2796,8 +2797,9 @@ class TestImportBuild:
assert data["error"] == "Unprocessable Entity"
assert re.match(
r"The modulemd .* is invalid\. Please verify the syntax is correct", data["message"]
r"The modulemd .* is invalid\. ", data["message"]
)
assert "Please verify the syntax is correct" in data["message"]
@pytest.mark.usefixtures("provide_test_client")