modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-27 12:12:22 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #922 Generate 'context' from hash based on buildrequires/requires stream. Reuse components only from module with the same stream_build_context.

Browse Source
This commit is contained in:
Matt Prahl
2018-04-24 22:50:02 +00:00
parent 9236133cd2 8d1d439737
commit 8351713828
17 changed files with 186 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/__init__.py
View File

@@ -121,6 +121,7 @@ def _populate_data(session, data_size=10, contexts=False):
build_one.name, build_one.stream, build_one.version, context)).hexdigest()
build_one.build_context = unique_hash
build_one.runtime_context = unique_hash
build_one.ref_build_context = unique_hash
with open(os.path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:
build_one.modulemd = mmd.read()
build_one.koji_tag = 'module-nginx-1.2'
@@ -399,8 +400,9 @@ def reuse_component_init_data():
build_one.stream = 'master'
build_one.version = 20170109091357
build_one.state = BUILD_STATES['ready']
build_one.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_one.ref_build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_one.runtime_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_one.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1'
build_one.koji_tag = 'module-de3adf79caf3e1b8'
build_one.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79'
build_one.batch = 3
@@ -482,8 +484,9 @@ def reuse_component_init_data():
build_two.stream = 'master'
build_two.version = 20170219191323
build_two.state = BUILD_STATES['build']
build_two.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_two.ref_build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_two.runtime_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_two.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1'
build_two.koji_tag = 'module-fe3adf73caf3e1b7'
build_two.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a'
build_two.batch = 1

2
tests/conftest.py
View File

@@ -146,7 +146,7 @@ def pdc_module_inactive(pdc):
'variant_type': 'module',
'variant_version': 'master',
'variant_release': '20180205135154',
'variant_context': 'c2c572ec',
'variant_context': '9c690d0e',
'koji_tag': 'module-95b214a704c984be',
'modulemd': TESTMODULE_MODULEMD,
'runtime_deps': [

2
tests/staged_data/formatted_testmodule-more-components.yaml
View File

@@ -4,7 +4,7 @@ data:
name: testmodule-more-components
stream: master
version: 20180205135154
context: c2c572ec
context: 9c690d0e
summary: A test module in all its beautiful beauty
description: This module demonstrates how to write simple modulemd files And can
be used for testing the build and release pipeline.

2
tests/staged_data/formatted_testmodule.yaml
View File

@@ -4,7 +4,7 @@ data:
name: testmodule
stream: master
version: 20180205135154
context: c2c572ec
context: 9c690d0e
summary: A test module in all its beautiful beauty
description: This module demonstrates how to write simple modulemd files And can
be used for testing the build and release pipeline.

3
tests/staged_data/testmodule_dependencies.yaml
View File

@@ -41,10 +41,13 @@ data:
buildrequires:
zebra:
ref: 08f8671f5925ecbd7c55d83e8368dd0be81ef8ed
stream: 2.0
base-runtime:
ref: ae993ba84f4bce554471382ccba917ef16265f11
stream: f28
fluffy:
ref: ea83325b231f64c575ea104bb698e9d43b80469a
stream: foo
requires:
fidget:
ref: 5aa32bd61aadaec8295ef90f79daaf190e387509

32
tests/test_build/test_build.py
View File

@@ -773,9 +773,9 @@ class TestBuild:
# Check that components are tagged after the batch is built.
tag_groups = []
tag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
if dest_tag is True:
@@ -784,9 +784,9 @@ class TestBuild:
buildtag_groups = []
buildtag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
assert buildtag_groups.pop(0) == set(artifacts)
@@ -829,9 +829,9 @@ class TestBuild:
# Check that components are tagged after the batch is built.
tag_groups = []
tag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
if dest_tag is True:
@@ -840,9 +840,9 @@ class TestBuild:
buildtag_groups = []
buildtag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
assert buildtag_groups.pop(0) == set(artifacts)
@@ -879,7 +879,8 @@ class TestBuild:
build_one.stream = 'master'
build_one.version = 20180205135154
build_one.build_context = 'return_runtime_context'
build_one.runtime_context = 'c7b355af'
build_one.ref_build_context = 'return_runtime_context'
build_one.runtime_context = '9c690d0e'
build_one.state = models.BUILD_STATES['failed']
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(
@@ -911,7 +912,7 @@ class TestBuild:
component_one.format = 'rpms'
component_one.scmurl = 'git://pkgs.stg.fedoraproject.org/rpms/perl-Tangerine.git?#master'
component_one.state = koji.BUILD_STATES['COMPLETE']
component_one.nvr = 'perl-Tangerine-0:0.22-2.module+0+814cfa39'
component_one.nvr = 'perl-Tangerine-0:0.22-2.module+0+a43e2001'
component_one.batch = 2
component_one.module_id = 2
component_one.ref = '7e96446223f1ad84a26c7cf23d6591cd9f6326c6'
@@ -1006,7 +1007,8 @@ class TestBuild:
build_one.stream = 'master'
build_one.version = 20180205135154
build_one.build_context = 'return_runtime_context'
build_one.runtime_context = 'c7b355af'
build_one.ref_build_context = 'return_runtime_context'
build_one.runtime_context = '9c690d0e'
build_one.state = models.BUILD_STATES['failed']
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(

8
tests/test_models/test_models.py
View File

@@ -67,10 +67,12 @@ class TestModels:
mmd = Modulemd.Module.new_from_file(yaml_path)
mmd.upgrade()
build.modulemd = mmd.dumps()
build.build_context, build.runtime_context = ModuleBuild.contexts_from_mmd(build.modulemd)
assert build.build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7'
build.ref_build_context, build.build_context, build.runtime_context = \
ModuleBuild.contexts_from_mmd(build.modulemd)
assert build.ref_build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7'
assert build.build_context == '089df24993c037e10174f3fa7342ab4dc191a4d4'
assert build.runtime_context == 'bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c'
assert build.context == 'f1a17afd'
assert build.context == '3ee22b28'
def test_siblings_property(self):
""" Tests that the siblings property returns the ID of all modules with

5
tests/test_resolver/test_db.py
View File

@@ -44,6 +44,7 @@ class TestDBModule:
Tests that the buildrequires of testmodule are returned
"""
expected = set(['module-f28-build'])
module = models.ModuleBuild.query.get(2)
if empty_buildrequires:
expected = set()
module = models.ModuleBuild.query.get(2)
@@ -58,7 +59,7 @@ class TestDBModule:
db.session.commit()
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
result = resolver.get_module_build_dependencies(
'testmodule', 'master', '20170109091357', '7c29193d').keys()
'testmodule', 'master', '20170109091357', 'c40c156c').keys()
assert set(result) == expected
def test_get_module_build_dependencies_recursive(self):
@@ -89,7 +90,7 @@ class TestDBModule:
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
result = resolver.get_module_build_dependencies(
'testmodule2', 'master', '20180123171545', '7c29193d').keys()
'testmodule2', 'master', '20180123171545', 'c40c156c').keys()
assert set(result) == set(['module-f28-build'])
@patch("module_build_service.config.Config.system",

8
tests/test_resolver/test_pdc.py
View File

@@ -40,9 +40,9 @@ class TestPDCModule:
def test_get_module_modulemds_nsvc(self, pdc_module_active_two_contexts):
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
ret = resolver.get_module_modulemds('testmodule', 'master', '20180205135154', 'c2c572ec')
ret = resolver.get_module_modulemds('testmodule', 'master', '20180205135154', '9c690d0e')
nsvcs = set(m.dup_nsvc() for m in ret)
expected = set(["testmodule:master:125a91f56532:c2c572ec"])
expected = set(["testmodule:master:125a91f56532:9c690d0e"])
assert nsvcs == expected
@pytest.mark.parametrize('kwargs', [{'version': '20180205135154'}, {}])
@@ -50,7 +50,7 @@ class TestPDCModule:
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
ret = resolver.get_module_modulemds('testmodule', 'master', **kwargs)
nsvcs = set(m.dup_nsvc() for m in ret)
expected = set(["testmodule:master:125a91f56532:c2c572ec",
expected = set(["testmodule:master:125a91f56532:9c690d0e",
"testmodule:master:125a91f56532:c2c572ed"])
assert nsvcs == expected
@@ -75,7 +75,7 @@ class TestPDCModule:
})
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
result = resolver.get_module_build_dependencies(
'testmodule', 'master', '20180205135154', 'c2c572ec').keys()
'testmodule', 'master', '20180205135154', '9c690d0e').keys()
assert set(result) == expected
def test_resolve_profiles(self, pdc_module_active):

8
tests/test_scheduler/test_poller.py
View File

@@ -395,10 +395,10 @@ class TestPoller:
assert module_build_two.state == models.BUILD_STATES['failed']
# Make sure the builds were untagged
builder.untag_artifacts.assert_called_once_with([
'perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39',
'module-build-macros-0.1-1.module+0+814cfa39'
'perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001',
'module-build-macros-0.1-1.module+0+a43e2001'
])
def test_cleanup_stale_failed_builds_no_components(self, create_builder, koji_get_session,

47
tests/test_utils/test_utils.py
View File

@@ -143,7 +143,10 @@ class TestUtilsComponentReuse:
db.session, second_module_build, 'perl-Tangerine')
assert pt_rv is None
def test_get_reusable_component_different_buildrequires_hash(self):
@pytest.mark.parametrize('rebuild_strategy', models.ModuleBuild.rebuild_strategies.keys())
def test_get_reusable_component_different_buildrequires_hash(self, rebuild_strategy):
first_module_build = models.ModuleBuild.query.filter_by(id=2).one()
first_module_build.rebuild_strategy = rebuild_strategy
second_module_build = models.ModuleBuild.query.filter_by(id=3).one()
mmd = second_module_build.mmd()
xmd = glib.from_variant_dict(mmd.get_xmd())
@@ -151,19 +154,49 @@ class TestUtilsComponentReuse:
'da39a3ee5e6b4b0d3255bfef95601890afd80709'
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.rebuild_strategy = rebuild_strategy
db.session.commit()
plc_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-List-Compare')
assert plc_rv is None
pt_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-Tangerine')
assert pt_rv is None
tangerine_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'tangerine')
if rebuild_strategy == "only-changed":
assert plc_rv is not None
assert pt_rv is not None
assert tangerine_rv is not None
else:
assert plc_rv is None
assert pt_rv is None
assert tangerine_rv is None
@pytest.mark.parametrize('rebuild_strategy', models.ModuleBuild.rebuild_strategies.keys())
def test_get_reusable_component_different_buildrequires_stream(self, rebuild_strategy):
first_module_build = models.ModuleBuild.query.filter_by(id=2).one()
first_module_build.rebuild_strategy = rebuild_strategy
second_module_build = models.ModuleBuild.query.filter_by(id=3).one()
mmd = second_module_build.mmd()
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['buildrequires']['platform']['stream'] = 'different'
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.rebuild_strategy = rebuild_strategy
db.session.commit()
plc_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-List-Compare')
pt_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-Tangerine')
tangerine_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'tangerine')
assert plc_rv is None
assert pt_rv is None
assert tangerine_rv is None
def test_get_reusable_component_different_buildrequires(self):
@@ -182,7 +215,7 @@ class TestUtilsComponentReuse:
}
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
db.session.commit()
plc_rv = module_build_service.utils.get_reusable_component(

3
tests/test_utils/test_utils_mse.py
View File

@@ -116,6 +116,7 @@ class TestUtilsModuleStreamExpansion:
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = 'changed-and-after'
module_build.build_context = context
module_build.stream_build_context = context
module_build.runtime_context = context
module_build.modulemd = mmd.dumps()
db.session.add(module_build)
@@ -160,7 +161,7 @@ class TestUtilsModuleStreamExpansion:
mmds = module_build_service.utils.generate_expanded_mmds(
db.session, module_build.mmd())
contexts = set([mmd.get_context() for mmd in mmds])
assert set(['3031e5a5', '6d10e00e']) == contexts
assert set(['ea432ace', 'b613fe68']) == contexts
@pytest.mark.parametrize(
'requires,build_requires,stream_ambigous,expected_xmd,expected_buildrequires', [