modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-05-11 18:37:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Pass buildrequired modules built against all compatible base module streams to MMDResolver.

Browse Source 
Imagine we have "platform:f29.0.0" and "platform:f29.1.0" base modules.
We also have "DBI" module we want to build agaisnt "platform:f29.1.0".
This "DBI" module depends on "perl" module which is only build against
"platform:f29.0.0".

Currently, DBI build would fail to resolve the dependencies, because
it wouldn't find "perl" module, because it is built against different
platform stream.

This PR changes the MSE code to include buildrequired module builds built
against all the compatible platform streams.

It does so by introducing following changes:

- MSE code uses new get_base_module_mmds() method to find out all the
  compatible platform modules. This needed new methods in DBResolver
  and MBSResolver.
- For each buildrequired module defined by name:stream, the MSE code then
  finds particular NSVC built against each compatible platform module.

Side effect of these code changes is that every module now must buildrequire
some base module.
This commit is contained in:
Jan Kaluza
2018-10-24 13:33:18 +02:00
parent 6b496dfde4
commit f2a236bc74
12 changed files with 423 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tests/__init__.py
View File

@@ -93,15 +93,27 @@ def clean_database(add_platform_module=True):
import_mmd(db.session, mmd)
def init_data(data_size=10, contexts=False):
def init_data(data_size=10, contexts=False, multiple_stream_versions=False):
"""
Creates data_size * 3 modules in database in different states and
with different component builds. See _populate_data for more info.
:param bool contexts: If True, multiple streams and contexts in each stream
are generated for 'nginx' module.
:param bool multiple_stream_versions: If true, multiple base modules with
difference stream versions are generated.
"""
clean_database()
if multiple_stream_versions:
mmd = load_mmd(os.path.join(base_dir, 'staged_data', 'platform.yaml'), True)
for stream in ["f28.0.0", "f29.0.0", "f29.1.0", "f29.2.0"]:
mmd.set_name("platform")
mmd.set_stream(stream)
import_mmd(db.session, mmd)
# Just to possibly confuse tests by adding another base module.
mmd.set_name("bootstrap")
mmd.set_stream(stream)
import_mmd(db.session, mmd)
with make_session(conf) as session:
_populate_data(session, data_size, contexts=contexts)
@@ -681,7 +693,7 @@ def reuse_shared_userspace_init_data():
session.add(build)
def make_module(nsvc, requires_list, build_requires_list):
def make_module(nsvc, requires_list, build_requires_list, base_module=None):
"""
Creates new models.ModuleBuild defined by `nsvc` string with requires
and buildrequires set according to `requires_list` and `build_requires_list`.
@@ -734,6 +746,7 @@ def make_module(nsvc, requires_list, build_requires_list):
module_build = ModuleBuild()
module_build.name = name
module_build.stream = stream
module_build.stream_version = module_build.get_stream_version(stream)
module_build.version = version
module_build.context = context
module_build.state = BUILD_STATES['ready']
@@ -747,6 +760,8 @@ def make_module(nsvc, requires_list, build_requires_list):
module_build.stream_build_context = context
module_build.runtime_context = context
module_build.modulemd = mmd.dumps()
if base_module:
module_build.buildrequires.append(base_module)
db.session.add(module_build)
db.session.commit()

5
tests/staged_data/fakemodule.yaml
View File

@@ -8,3 +8,8 @@ data:
module:
- MIT
content: []
dependencies:
buildrequires:
platform: f28
requires:
platform: f28

1
tests/staged_data/testmodule-no-deps.yaml
View File

@@ -9,6 +9,7 @@ data:
module: [ MIT ]
dependencies:
buildrequires:
platform: f28
chineese_food: good
requires:
noodles: lomein

11
tests/test_models/test_models.py
View File

@@ -23,7 +23,7 @@
import os
from tests.test_models import init_data, module_build_from_modulemd
from tests import init_data as init_data_contexts, clean_database
from tests import (init_data as init_data_contexts, clean_database)
from module_build_service import conf, Modulemd
from module_build_service.models import ComponentBuild, ModuleBuild, make_session
@@ -132,3 +132,12 @@ class TestModelsGetStreamsContexts:
builds = ["%s:%s:%s:%s" % (build.name, build.stream, str(build.version),
build.context) for build in builds]
assert builds == ['nginx:1:3:d5a6c0fa', 'nginx:1:3:795e97c1']
def test_get_last_builds_in_stream_version_lte(self):
init_data_contexts(1, multiple_stream_versions=True)
with make_session(conf) as session:
builds = ModuleBuild.get_last_builds_in_stream_version_lte(
session, "platform", 290100)
builds = set(["%s:%s:%s:%s" % (build.name, build.stream, str(build.version),
build.context) for build in builds])
assert builds == set(['platform:f29.0.0:3:00000000', 'platform:f29.1.0:3:00000000'])

51
tests/test_resolver/test_db.py
View File

@@ -22,11 +22,14 @@
import os
from datetime import datetime
from mock import patch, PropertyMock
import pytest
import module_build_service.resolver as mbs_resolver
from module_build_service import app, db, models, glib, utils, Modulemd
from module_build_service.utils import import_mmd, load_mmd
from module_build_service.models import ModuleBuild
import tests
@@ -38,6 +41,54 @@ class TestDBModule:
def setup_method(self):
tests.reuse_component_init_data()
def test_get_buildrequired_modulemds(self):
mmd = load_mmd(os.path.join(base_dir, 'staged_data', 'platform.yaml'), True)
mmd.set_stream('f30.1.3')
import_mmd(db.session, mmd)
platform_f300103 = ModuleBuild.query.filter_by(stream='f30.1.3').one()
mmd.set_name("testmodule")
mmd.set_stream("master")
mmd.set_version(20170109091357)
mmd.set_context("123")
build = ModuleBuild(
name='testmodule',
stream='master',
version=20170109091357,
state=5,
build_context='dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3',
runtime_context='ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7',
context='7c29193d',
koji_tag='module-testmodule-master-20170109091357-7c29193d',
scmurl='git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79',
batch=3,
owner='Dr. Pepper',
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy='changed-and-after',
modulemd=mmd.dumps()
)
build.buildrequires.append(platform_f300103)
db.session.add(build)
db.session.commit()
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
result = resolver.get_buildrequired_modulemds(
"testmodule", "master", platform_f300103.mmd().dup_nsvc())
nsvcs = set([m.dup_nsvc() for m in result])
assert nsvcs == set(['testmodule:master:20170109091357:123'])
@pytest.mark.parametrize('stream_versions', [False, True])
def test_get_module_modulemds_stream_versions(self, stream_versions):
tests.init_data(1, multiple_stream_versions=True)
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
result = resolver.get_module_modulemds(
"platform", "f29.1.0", stream_version_lte=stream_versions)
nsvcs = set([mmd.dup_nsvc() for mmd in result])
if stream_versions:
assert nsvcs == set(['platform:f29.1.0:3:00000000', 'platform:f29.0.0:3:00000000'])
else:
assert nsvcs == set(['platform:f29.1.0:3:00000000'])
@pytest.mark.parametrize('empty_buildrequires', [False, True])
def test_get_module_build_dependencies(self, empty_buildrequires):
"""

159
tests/test_utils/test_utils_mse.py
View File

@@ -52,22 +52,22 @@ class TestUtilsModuleStreamExpansion:
Generates gtk:1, gtk:2, foo:1 and foo:2 modules requiring the
platform:f28 and platform:f29 modules.
"""
make_module("gtk:1:0:c2", {"platform": ["f28"]}, {})
make_module("gtk:1:0:c3", {"platform": ["f29"]}, {})
make_module("gtk:2:0:c4", {"platform": ["f28"]}, {})
make_module("gtk:2:0:c5", {"platform": ["f29"]}, {})
make_module("foo:1:0:c2", {"platform": ["f28"]}, {})
make_module("foo:1:0:c3", {"platform": ["f29"]}, {})
make_module("foo:2:0:c4", {"platform": ["f28"]}, {})
make_module("foo:2:0:c5", {"platform": ["f29"]}, {})
make_module("platform:f28:0:c10", {}, {})
make_module("platform:f29:0:c11", {}, {})
make_module("app:1:0:c6", {"platform": ["f29"]}, {})
platform_f28 = make_module("platform:f28:0:c10", {}, {})
platform_f29 = make_module("platform:f29:0:c11", {}, {})
make_module("gtk:1:0:c2", {"platform": ["f28"]}, {}, platform_f28)
make_module("gtk:1:0:c3", {"platform": ["f29"]}, {}, platform_f29)
make_module("gtk:2:0:c4", {"platform": ["f28"]}, {}, platform_f28)
make_module("gtk:2:0:c5", {"platform": ["f29"]}, {}, platform_f29)
make_module("foo:1:0:c2", {"platform": ["f28"]}, {}, platform_f28)
make_module("foo:1:0:c3", {"platform": ["f29"]}, {}, platform_f29)
make_module("foo:2:0:c4", {"platform": ["f28"]}, {}, platform_f28)
make_module("foo:2:0:c5", {"platform": ["f29"]}, {}, platform_f29)
make_module("app:1:0:c6", {"platform": ["f29"]}, {}, platform_f29)
def test_generate_expanded_mmds_context(self):
self._generate_default_modules()
module_build = make_module(
"app:1:0:c1", {"gtk": ["1", "2"]}, {"gtk": ["1", "2"]})
"app:1:0:c1", {"gtk": ["1", "2"]}, {"platform": ["f28"], "gtk": ["1", "2"]})
mmds = module_build_service.utils.generate_expanded_mmds(
db.session, module_build.mmd())
contexts = set([mmd.get_context() for mmd in mmds])
@@ -75,35 +75,30 @@ class TestUtilsModuleStreamExpansion:
@pytest.mark.parametrize(
'requires,build_requires,stream_ambigous,expected_xmd,expected_buildrequires', [
({"gtk": ["1", "2"]}, {"gtk": ["1", "2"]}, True,
({"gtk": ["1", "2"]},
{"platform": ["f28"], "gtk": ["1", "2"]}, True,
set([
frozenset(['platform:f28:0:c10', 'gtk:2:0:c4']),
frozenset(['platform:f28:0:c10', 'gtk:1:0:c2'])
]),
set([
frozenset(['gtk:1']),
frozenset(['gtk:2']),
frozenset(['gtk:1', 'platform:f28']),
frozenset(['gtk:2', 'platform:f28']),
])),
({"foo": ["1"]}, {"foo": ["1"], "gtk": ["1", "2"]}, True,
({"foo": ["1"]},
{"platform": ["f28"], "foo": ["1"], "gtk": ["1", "2"]}, True,
set([
frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10']),
frozenset(['foo:1:0:c2', 'gtk:2:0:c4', 'platform:f28:0:c10'])
]),
set([
frozenset(['foo:1', 'gtk:1']),
frozenset(['foo:1', 'gtk:2'])
frozenset(['foo:1', 'gtk:1', 'platform:f28']),
frozenset(['foo:1', 'gtk:2', 'platform:f28'])
])),
({"gtk": ["1"], "foo": ["1"]}, {"gtk": ["1"], "foo": ["1"]}, False,
set([
frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10'])
]),
set([
frozenset(['foo:1', 'gtk:1'])
])),
({"gtk": ["1"], "foo": ["1"]}, {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]}, False,
({"gtk": ["1"], "foo": ["1"]},
{"platform": ["f28"], "gtk": ["1"], "foo": ["1"]}, False,
set([
frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10'])
]),
@@ -111,44 +106,56 @@ class TestUtilsModuleStreamExpansion:
frozenset(['foo:1', 'gtk:1', 'platform:f28'])
])),
({"gtk": ["-2"], "foo": ["-2"]}, {"gtk": ["-2"], "foo": ["-2"]}, True,
({"gtk": ["1"], "foo": ["1"]},
{"gtk": ["1"], "foo": ["1"], "platform": ["f28"]}, False,
set([
frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10'])
]),
set([
frozenset(['foo:1', 'gtk:1'])
frozenset(['foo:1', 'gtk:1', 'platform:f28'])
])),
({"gtk": ["1"], "foo": ["1"]}, {"gtk": ["-1", "1"], "foo": ["-2", "1"]}, False,
({"gtk": ["-2"], "foo": ["-2"]},
{"platform": ["f28"], "gtk": ["-2"], "foo": ["-2"]}, True,
set([
frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10'])
]),
set([
frozenset(['foo:1', 'gtk:1'])
frozenset(['foo:1', 'gtk:1', 'platform:f28'])
])),
({"gtk": ["1"], "foo": ["1"]}, {"gtk": ["1"]}, False,
({"gtk": ["1"], "foo": ["1"]},
{"platform": ["f28"], "gtk": ["-1", "1"], "foo": ["-2", "1"]}, False,
set([
frozenset(['foo:1:0:c2', 'gtk:1:0:c2', 'platform:f28:0:c10'])
]),
set([
frozenset(['foo:1', 'gtk:1', 'platform:f28'])
])),
({"gtk": ["1"], "foo": ["1"]},
{"platform": ["f28"], "gtk": ["1"]}, False,
set([
frozenset(['gtk:1:0:c2', 'platform:f28:0:c10'])
]),
set([
frozenset(['gtk:1'])
frozenset(['gtk:1', 'platform:f28'])
])),
({"gtk": []}, {"gtk": ["1"]}, True,
({"gtk": []}, {"platform": ["f28"], "gtk": ["1"]}, True,
set([
frozenset(['gtk:1:0:c2', 'platform:f28:0:c10'])
]),
set([
frozenset(['gtk:1'])
frozenset(['gtk:1', 'platform:f28'])
])),
({}, {"app": ["1"]}, False,
({}, {"platform": ["f29"], "app": ["1"]}, False,
set([
frozenset(['app:1:0:c6', 'platform:f29:0:c11'])
]),
set([
frozenset(['app:1'])
frozenset(['app:1', 'platform:f29'])
])),
])
def test_generate_expanded_mmds_buildrequires(
@@ -204,33 +211,36 @@ class TestUtilsModuleStreamExpansion:
assert buildrequires_per_mmd_buildrequires == expected_buildrequires
@pytest.mark.parametrize('requires,build_requires,expected', [
({"gtk": ["1", "2"]}, {"gtk": ["1", "2"]},
({"gtk": ["1", "2"]}, {"platform": [], "gtk": ["1", "2"]},
set([
frozenset(['gtk:1']),
frozenset(['gtk:2']),
])),
({"gtk": ["1", "2"]}, {"gtk": ["1"]},
({"gtk": ["1", "2"]}, {"platform": [], "gtk": ["1"]},
set([
frozenset(['gtk:1', 'gtk:2']),
])),
({"gtk": ["1"], "foo": ["1"]}, {"gtk": ["1"], "foo": ["1"]},
({"gtk": ["1"], "foo": ["1"]},
{"platform": [], "gtk": ["1"], "foo": ["1"]},
set([
frozenset(['foo:1', 'gtk:1']),
])),
({"gtk": ["-2"], "foo": ["-2"]}, {"gtk": ["-2"], "foo": ["-2"]},
({"gtk": ["-2"], "foo": ["-2"]},
{"platform": [], "gtk": ["-2"], "foo": ["-2"]},
set([
frozenset(['foo:1', 'gtk:1']),
])),
({"gtk": ["-1", "1"], "foo": ["-2", "1"]}, {"gtk": ["-1", "1"], "foo": ["-2", "1"]},
({"gtk": ["-1", "1"], "foo": ["-2", "1"]},
{"platform": [], "gtk": ["-1", "1"], "foo": ["-2", "1"]},
set([
frozenset(['foo:1', 'gtk:1']),
])),
({"gtk": [], "foo": []}, {"gtk": ["1"], "foo": ["1"]},
({"gtk": [], "foo": []}, {"platform": [], "gtk": ["1"], "foo": ["1"]},
set([
frozenset([]),
])),
@@ -255,28 +265,29 @@ class TestUtilsModuleStreamExpansion:
assert requires_per_mmd == expected
@pytest.mark.parametrize('requires,build_requires,expected', [
({}, {"gtk": ["1", "2"]},
({}, {"platform": [], "gtk": ["1", "2"]},
['platform:f29:0:c11', 'gtk:2:0:c4', 'gtk:2:0:c5',
'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']),
({}, {"gtk": ["1"], "foo": ["1"]},
({}, {"platform": [], "gtk": ["1"], "foo": ["1"]},
['platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3',
'foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11']),
({}, {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]},
['platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3',
'foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11']),
['platform:f28:0:c10', 'gtk:1:0:c2',
'foo:1:0:c2']),
([{}, {}], [{"gtk": ["1"], "foo": ["1"]}, {"gtk": ["2"], "foo": ["2"]}],
([{}, {}], [{"platform": [], "gtk": ["1"], "foo": ["1"]},
{"platform": [], "gtk": ["2"], "foo": ["2"]}],
['foo:1:0:c2', 'foo:1:0:c3', 'foo:2:0:c4', 'foo:2:0:c5',
'platform:f28:0:c10', 'platform:f29:0:c11', 'gtk:1:0:c2',
'gtk:1:0:c3', 'gtk:2:0:c4', 'gtk:2:0:c5']),
({}, {"gtk": ["-2"], "foo": ["-2"]},
({}, {"platform": [], "gtk": ["-2"], "foo": ["-2"]},
['foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11',
'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']),
({}, {"gtk": ["-1", "1"], "foo": ["-2", "1"]},
({}, {"platform": [], "gtk": ["-1", "1"], "foo": ["-2", "1"]},
['foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11',
'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']),
])
@@ -292,23 +303,23 @@ class TestUtilsModuleStreamExpansion:
and lorem:1 modules which require base:f29 module requiring
platform:f29 module :).
"""
make_module("gtk:1:0:c2", {"foo": ["unknown"]}, {})
make_module("gtk:1:1:c2", {"foo": ["1"]}, {})
make_module("foo:1:0:c2", {"bar": ["unknown"]}, {})
make_module("foo:1:1:c2", {"bar": ["1"], "lorem": ["1"]}, {})
make_module("bar:1:0:c2", {"base": ["unknown"]}, {})
make_module("bar:1:1:c2", {"base": ["f29"]}, {})
make_module("lorem:1:0:c2", {"base": ["unknown"]}, {})
make_module("lorem:1:1:c2", {"base": ["f29"]}, {})
make_module("base:f29:0:c3", {"platform": ["f29"]}, {})
make_module("platform:f29:0:c11", {}, {})
base_module = make_module("platform:f29:0:c11", {}, {})
make_module("gtk:1:0:c2", {"foo": ["unknown"]}, {}, base_module)
make_module("gtk:1:1:c2", {"foo": ["1"]}, {}, base_module)
make_module("foo:1:0:c2", {"bar": ["unknown"]}, {}, base_module)
make_module("foo:1:1:c2", {"bar": ["1"], "lorem": ["1"]}, {}, base_module)
make_module("bar:1:0:c2", {"base": ["unknown"]}, {}, base_module)
make_module("bar:1:1:c2", {"base": ["f29"]}, {}, base_module)
make_module("lorem:1:0:c2", {"base": ["unknown"]}, {}, base_module)
make_module("lorem:1:1:c2", {"base": ["f29"]}, {}, base_module)
make_module("base:f29:0:c3", {"platform": ["f29"]}, {}, base_module)
@pytest.mark.parametrize('requires,build_requires,expected', [
({}, {"gtk": ["1"]},
({}, {"platform": [], "gtk": ["1"]},
['foo:1:1:c2', 'base:f29:0:c3', 'platform:f29:0:c11',
'bar:1:1:c2', 'gtk:1:1:c2', 'lorem:1:1:c2']),
({}, {"foo": ["1"]},
({}, {"platform": [], "foo": ["1"]},
['foo:1:1:c2', 'base:f29:0:c3', 'platform:f29:0:c11',
'bar:1:1:c2', 'lorem:1:1:c2']),
])
@@ -317,3 +328,27 @@ class TestUtilsModuleStreamExpansion:
self._generate_default_modules_recursion()
nsvcs = self._get_mmds_required_by_module_recursively(module_build)
assert set(nsvcs) == set(expected)
def _generate_default_modules_modules_multiple_stream_versions(self):
"""
Generates the gtk:1 module requiring foo:1 module requiring bar:1
and lorem:1 modules which require base:f29 module requiring
platform:f29 module :).
"""
f290000 = make_module("platform:f29.0.0:0:c11", {}, {})
f290100 = make_module("platform:f29.1.0:0:c11", {}, {})
f290200 = make_module("platform:f29.2.0:0:c11", {}, {})
make_module("gtk:1:0:c2", {"platform": ["f29"]}, {}, f290000)
make_module("gtk:1:1:c2", {"platform": ["f29"]}, {}, f290100)
make_module("gtk:1:2:c2", {"platform": ["f29"]}, {}, f290100)
make_module("gtk:1:3:c2", {"platform": ["f29"]}, {}, f290200)
@pytest.mark.parametrize('requires,build_requires,expected', [
({}, {"platform": ["f29.1.0"], "gtk": ["1"]},
['platform:f29.0.0:0:c11', 'gtk:1:0:c2', 'gtk:1:2:c2', 'platform:f29.1.0:0:c11']),
])
def test_get_required_modules_stream_versions(self, requires, build_requires, expected):
module_build = make_module("app:1:0:c1", requires, build_requires)
self._generate_default_modules_modules_multiple_stream_versions()
nsvcs = self._get_mmds_required_by_module_recursively(module_build)
assert set(nsvcs) == set(expected)

2
tests/test_views/test_views.py
View File

@@ -893,7 +893,7 @@ class TestViews:
assert data['name'] == 'fakemodule'
assert data['scmurl'] == ('git://pkgs.stg.fedoraproject.org/modules/testmodule.git'
'?#68931c90de214d9d13feefbd35246a81b6cb8d49')
assert data['version'] == '1'
assert data['version'] == '281'
assert data['time_submitted'] is not None
assert data['time_modified'] is not None
assert data['time_completed'] is None