modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 03:38:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #922 Generate 'context' from hash based on buildrequires/requires stream. Reuse components only from module with the same stream_build_context.

Browse Source
This commit is contained in:
Matt Prahl
2018-04-24 22:50:02 +00:00
parent 9236133cd2 8d1d439737
commit 8351713828
17 changed files with 186 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
module_build_service/migrations/versions/caeae7a4f537_ref_build_context.py Normal file
View File

@@ -0,0 +1,77 @@
"""Add ModuleBuid.ref_build_context.
Revision ID: caeae7a4f537
Revises: 9ca1c166f426
Create Date: 2018-04-18 13:37:40.365129
"""
# revision identifiers, used by Alembic.
revision = 'caeae7a4f537'
down_revision = '9ca1c166f426'
from alembic import op
import sqlalchemy as sa
# Data migration imports
from module_build_service import Modulemd
import hashlib
import json
from collections import OrderedDict
modulebuild = sa.Table(
'module_builds',
sa.MetaData(),
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('modulemd', sa.String()),
sa.Column('build_context', sa.String()),
sa.Column('runtime_context', sa.String()),
)
def upgrade():
connection = op.get_bind()
with op.batch_alter_table('module_builds') as b:
b.alter_column('build_context', new_column_name='ref_build_context')
op.add_column('module_builds', sa.Column('build_context', sa.String()))
# Determine what the contexts should be based on the modulemd
for build in connection.execute(modulebuild.select()):
if not build.modulemd:
continue
try:
mmd = Modulemd.Module().new_from_string(build.modulemd)
mmd.upgrade()
except Exception:
# If the modulemd isn't parseable then skip this build
continue
mbs_xmd = mmd.get_xmd().get('mbs', {})
# Skip the non-MSE builds, so the "context" will be set default one
# in models.ModuleBuild.
if "mse" not in mbs_xmd.keys() or not mbs_xmd["mse"]:
continue
# It's possible this module build was built before MBS filled out xmd or before MBS
# filled out the requires in xmd. We also have to use keys because GLib.Variant
# doesn't support `in` directly.
if 'buildrequires' not in mbs_xmd.keys():
continue
# Get the streams of buildrequires and hash it.
mmd_formatted_buildrequires = {
dep: info['stream'] for dep, info in mbs_xmd["buildrequires"].items()}
property_json = json.dumps(OrderedDict(sorted(mmd_formatted_buildrequires.items())))
context = hashlib.sha1(property_json).hexdigest()
# Update the database now
connection.execute(
modulebuild.update().where(modulebuild.c.id == build.id).values(
build_context=context))
def downgrade():
op.drop_column('module_builds', 'build_context')
with op.batch_alter_table('module_builds') as b:
b.alter_column('ref_build_context', new_column_name='build_context')

18
module_build_service/models.py
View File

@@ -170,6 +170,7 @@ class ModuleBuild(MBSBase):
name = db.Column(db.String, nullable=False)
stream = db.Column(db.String, nullable=False)
version = db.Column(db.String, nullable=False)
ref_build_context = db.Column(db.String)
build_context = db.Column(db.String)
runtime_context = db.Column(db.String)
state = db.Column(db.Integer, nullable=False)
@@ -345,6 +346,16 @@ class ModuleBuild(MBSBase):
@staticmethod
def contexts_from_mmd(mmd_str):
"""
Returns tuple (ref_build_context, build_context, runtime_context) with hashes:
- ref_build_context - Hash of commit hashes of expanded buildrequires.
- build_context - Hash of stream names of expanded buildrequires.
- runtime_context - Hash of stream names of expanded runtime requires.
:param str mmd_str: String with Modulemd metadata.
:rtype: tuple of strings
:return: Tuple with build_context, strem_build_context and runtime_context hashes.
"""
try:
mmd = Modulemd.Module().new_from_string(mmd_str)
mmd.upgrade()
@@ -363,6 +374,12 @@ class ModuleBuild(MBSBase):
property_json = json.dumps(OrderedDict(sorted(mmd_formatted_buildrequires.items())))
rv.append(hashlib.sha1(property_json).hexdigest())
# Get the streams of buildrequires and hash it.
mmd_formatted_buildrequires = {
dep: info['stream'] for dep, info in mbs_xmd["buildrequires"].items()}
property_json = json.dumps(OrderedDict(sorted(mmd_formatted_buildrequires.items())))
rv.append(hashlib.sha1(property_json).hexdigest())
# Get the requires from the real "dependencies" section in MMD.
mmd_requires = {}
for deps in mmd.get_dependencies():
@@ -568,6 +585,7 @@ class ModuleBuild(MBSBase):
state_url = get_url_for('module_build', api_version=api_version, id=self.id)
json.update({
'component_builds': [build.id for build in self.component_builds],
'ref_build_context': self.ref_build_context,
'build_context': self.build_context,
'modulemd': self.modulemd,
'runtime_context': self.runtime_context,

3
module_build_service/utils/mse.py
View File

@@ -359,7 +359,8 @@ def generate_expanded_mmds(session, mmd, raise_if_stream_ambigous=False, default
mmd_copy.set_xmd(glib.dict_values(xmd))
# Now we have all the info to actually compute context of this module.
build_context, runtime_context = models.ModuleBuild.contexts_from_mmd(mmd_copy.dumps())
ref_build_context, build_context, runtime_context = \
models.ModuleBuild.contexts_from_mmd(mmd_copy.dumps())
context = models.ModuleBuild.context_from_contexts(build_context, runtime_context)
mmd_copy.set_context(context)

3
module_build_service/utils/reuse.py
View File

@@ -95,6 +95,7 @@ def _get_reusable_module(session, module):
.filter_by(stream=mmd.get_stream())\
.filter(models.ModuleBuild.state.in_([3, 5]))\
.filter(models.ModuleBuild.scmurl.isnot(None))\
.filter_by(build_context=module.build_context)\
.order_by(models.ModuleBuild.time_completed.desc())
# If we are rebuilding with the "changed-and-after" option, then we can't reuse
# components from modules that were built more liberally
@@ -102,7 +103,7 @@ def _get_reusable_module(session, module):
previous_module_build = previous_module_build.filter(
models.ModuleBuild.rebuild_strategy.in_(['all', 'changed-and-after']))
previous_module_build = previous_module_build.filter_by(
build_context=module.build_context)
ref_build_context=module.ref_build_context)
previous_module_build = previous_module_build.first()
# The component can't be reused if there isn't a previous build in the done
# or ready state

2
module_build_service/utils/submit.py
View File

@@ -377,7 +377,7 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
username=username,
**(optional_params or {})
)
module.build_context, module.runtime_context = \
module.ref_build_context, module.build_context, module.runtime_context = \
module.contexts_from_mmd(module.modulemd)
db.session.add(module)

7
tests/__init__.py
View File

@@ -121,6 +121,7 @@ def _populate_data(session, data_size=10, contexts=False):
build_one.name, build_one.stream, build_one.version, context)).hexdigest()
build_one.build_context = unique_hash
build_one.runtime_context = unique_hash
build_one.ref_build_context = unique_hash
with open(os.path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:
build_one.modulemd = mmd.read()
build_one.koji_tag = 'module-nginx-1.2'
@@ -399,8 +400,9 @@ def reuse_component_init_data():
build_one.stream = 'master'
build_one.version = 20170109091357
build_one.state = BUILD_STATES['ready']
build_one.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_one.ref_build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_one.runtime_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_one.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1'
build_one.koji_tag = 'module-de3adf79caf3e1b8'
build_one.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79'
build_one.batch = 3
@@ -482,8 +484,9 @@ def reuse_component_init_data():
build_two.stream = 'master'
build_two.version = 20170219191323
build_two.state = BUILD_STATES['build']
build_two.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_two.ref_build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_two.runtime_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'
build_two.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1'
build_two.koji_tag = 'module-fe3adf73caf3e1b7'
build_two.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a'
build_two.batch = 1

2
tests/conftest.py
View File

@@ -146,7 +146,7 @@ def pdc_module_inactive(pdc):
'variant_type': 'module',
'variant_version': 'master',
'variant_release': '20180205135154',
'variant_context': 'c2c572ec',
'variant_context': '9c690d0e',
'koji_tag': 'module-95b214a704c984be',
'modulemd': TESTMODULE_MODULEMD,
'runtime_deps': [

2
tests/staged_data/formatted_testmodule-more-components.yaml
View File

@@ -4,7 +4,7 @@ data:
name: testmodule-more-components
stream: master
version: 20180205135154
context: c2c572ec
context: 9c690d0e
summary: A test module in all its beautiful beauty
description: This module demonstrates how to write simple modulemd files And can
be used for testing the build and release pipeline.

2
tests/staged_data/formatted_testmodule.yaml
View File

@@ -4,7 +4,7 @@ data:
name: testmodule
stream: master
version: 20180205135154
context: c2c572ec
context: 9c690d0e
summary: A test module in all its beautiful beauty
description: This module demonstrates how to write simple modulemd files And can
be used for testing the build and release pipeline.

3
tests/staged_data/testmodule_dependencies.yaml
View File

@@ -41,10 +41,13 @@ data:
buildrequires:
zebra:
ref: 08f8671f5925ecbd7c55d83e8368dd0be81ef8ed
stream: 2.0
base-runtime:
ref: ae993ba84f4bce554471382ccba917ef16265f11
stream: f28
fluffy:
ref: ea83325b231f64c575ea104bb698e9d43b80469a
stream: foo
requires:
fidget:
ref: 5aa32bd61aadaec8295ef90f79daaf190e387509

32
tests/test_build/test_build.py
View File

@@ -773,9 +773,9 @@ class TestBuild:
# Check that components are tagged after the batch is built.
tag_groups = []
tag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
if dest_tag is True:
@@ -784,9 +784,9 @@ class TestBuild:
buildtag_groups = []
buildtag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
assert buildtag_groups.pop(0) == set(artifacts)
@@ -829,9 +829,9 @@ class TestBuild:
# Check that components are tagged after the batch is built.
tag_groups = []
tag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
if dest_tag is True:
@@ -840,9 +840,9 @@ class TestBuild:
buildtag_groups = []
buildtag_groups.append(set(
['perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39']))
['perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001']))
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
assert buildtag_groups.pop(0) == set(artifacts)
@@ -879,7 +879,8 @@ class TestBuild:
build_one.stream = 'master'
build_one.version = 20180205135154
build_one.build_context = 'return_runtime_context'
build_one.runtime_context = 'c7b355af'
build_one.ref_build_context = 'return_runtime_context'
build_one.runtime_context = '9c690d0e'
build_one.state = models.BUILD_STATES['failed']
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(
@@ -911,7 +912,7 @@ class TestBuild:
component_one.format = 'rpms'
component_one.scmurl = 'git://pkgs.stg.fedoraproject.org/rpms/perl-Tangerine.git?#master'
component_one.state = koji.BUILD_STATES['COMPLETE']
component_one.nvr = 'perl-Tangerine-0:0.22-2.module+0+814cfa39'
component_one.nvr = 'perl-Tangerine-0:0.22-2.module+0+a43e2001'
component_one.batch = 2
component_one.module_id = 2
component_one.ref = '7e96446223f1ad84a26c7cf23d6591cd9f6326c6'
@@ -1006,7 +1007,8 @@ class TestBuild:
build_one.stream = 'master'
build_one.version = 20180205135154
build_one.build_context = 'return_runtime_context'
build_one.runtime_context = 'c7b355af'
build_one.ref_build_context = 'return_runtime_context'
build_one.runtime_context = '9c690d0e'
build_one.state = models.BUILD_STATES['failed']
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(

8
tests/test_models/test_models.py
View File

@@ -67,10 +67,12 @@ class TestModels:
mmd = Modulemd.Module.new_from_file(yaml_path)
mmd.upgrade()
build.modulemd = mmd.dumps()
build.build_context, build.runtime_context = ModuleBuild.contexts_from_mmd(build.modulemd)
assert build.build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7'
build.ref_build_context, build.build_context, build.runtime_context = \
ModuleBuild.contexts_from_mmd(build.modulemd)
assert build.ref_build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7'
assert build.build_context == '089df24993c037e10174f3fa7342ab4dc191a4d4'
assert build.runtime_context == 'bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c'
assert build.context == 'f1a17afd'
assert build.context == '3ee22b28'
def test_siblings_property(self):
""" Tests that the siblings property returns the ID of all modules with

5
tests/test_resolver/test_db.py
View File

@@ -44,6 +44,7 @@ class TestDBModule:
Tests that the buildrequires of testmodule are returned
"""
expected = set(['module-f28-build'])
module = models.ModuleBuild.query.get(2)
if empty_buildrequires:
expected = set()
module = models.ModuleBuild.query.get(2)
@@ -58,7 +59,7 @@ class TestDBModule:
db.session.commit()
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
result = resolver.get_module_build_dependencies(
'testmodule', 'master', '20170109091357', '7c29193d').keys()
'testmodule', 'master', '20170109091357', 'c40c156c').keys()
assert set(result) == expected
def test_get_module_build_dependencies_recursive(self):
@@ -89,7 +90,7 @@ class TestDBModule:
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
result = resolver.get_module_build_dependencies(
'testmodule2', 'master', '20180123171545', '7c29193d').keys()
'testmodule2', 'master', '20180123171545', 'c40c156c').keys()
assert set(result) == set(['module-f28-build'])
@patch("module_build_service.config.Config.system",

8
tests/test_resolver/test_pdc.py
View File

@@ -40,9 +40,9 @@ class TestPDCModule:
def test_get_module_modulemds_nsvc(self, pdc_module_active_two_contexts):
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
ret = resolver.get_module_modulemds('testmodule', 'master', '20180205135154', 'c2c572ec')
ret = resolver.get_module_modulemds('testmodule', 'master', '20180205135154', '9c690d0e')
nsvcs = set(m.dup_nsvc() for m in ret)
expected = set(["testmodule:master:125a91f56532:c2c572ec"])
expected = set(["testmodule:master:125a91f56532:9c690d0e"])
assert nsvcs == expected
@pytest.mark.parametrize('kwargs', [{'version': '20180205135154'}, {}])
@@ -50,7 +50,7 @@ class TestPDCModule:
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
ret = resolver.get_module_modulemds('testmodule', 'master', **kwargs)
nsvcs = set(m.dup_nsvc() for m in ret)
expected = set(["testmodule:master:125a91f56532:c2c572ec",
expected = set(["testmodule:master:125a91f56532:9c690d0e",
"testmodule:master:125a91f56532:c2c572ed"])
assert nsvcs == expected
@@ -75,7 +75,7 @@ class TestPDCModule:
})
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
result = resolver.get_module_build_dependencies(
'testmodule', 'master', '20180205135154', 'c2c572ec').keys()
'testmodule', 'master', '20180205135154', '9c690d0e').keys()
assert set(result) == expected
def test_resolve_profiles(self, pdc_module_active):

8
tests/test_scheduler/test_poller.py
View File

@@ -395,10 +395,10 @@ class TestPoller:
assert module_build_two.state == models.BUILD_STATES['failed']
# Make sure the builds were untagged
builder.untag_artifacts.assert_called_once_with([
'perl-Tangerine-0.23-1.module+0+814cfa39',
'perl-List-Compare-0.53-5.module+0+814cfa39',
'tangerine-0.22-3.module+0+814cfa39',
'module-build-macros-0.1-1.module+0+814cfa39'
'perl-Tangerine-0.23-1.module+0+a43e2001',
'perl-List-Compare-0.53-5.module+0+a43e2001',
'tangerine-0.22-3.module+0+a43e2001',
'module-build-macros-0.1-1.module+0+a43e2001'
])
def test_cleanup_stale_failed_builds_no_components(self, create_builder, koji_get_session,

47
tests/test_utils/test_utils.py
View File

@@ -143,7 +143,10 @@ class TestUtilsComponentReuse:
db.session, second_module_build, 'perl-Tangerine')
assert pt_rv is None
def test_get_reusable_component_different_buildrequires_hash(self):
@pytest.mark.parametrize('rebuild_strategy', models.ModuleBuild.rebuild_strategies.keys())
def test_get_reusable_component_different_buildrequires_hash(self, rebuild_strategy):
first_module_build = models.ModuleBuild.query.filter_by(id=2).one()
first_module_build.rebuild_strategy = rebuild_strategy
second_module_build = models.ModuleBuild.query.filter_by(id=3).one()
mmd = second_module_build.mmd()
xmd = glib.from_variant_dict(mmd.get_xmd())
@@ -151,19 +154,49 @@ class TestUtilsComponentReuse:
'da39a3ee5e6b4b0d3255bfef95601890afd80709'
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.rebuild_strategy = rebuild_strategy
db.session.commit()
plc_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-List-Compare')
assert plc_rv is None
pt_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-Tangerine')
assert pt_rv is None
tangerine_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'tangerine')
if rebuild_strategy == "only-changed":
assert plc_rv is not None
assert pt_rv is not None
assert tangerine_rv is not None
else:
assert plc_rv is None
assert pt_rv is None
assert tangerine_rv is None
@pytest.mark.parametrize('rebuild_strategy', models.ModuleBuild.rebuild_strategies.keys())
def test_get_reusable_component_different_buildrequires_stream(self, rebuild_strategy):
first_module_build = models.ModuleBuild.query.filter_by(id=2).one()
first_module_build.rebuild_strategy = rebuild_strategy
second_module_build = models.ModuleBuild.query.filter_by(id=3).one()
mmd = second_module_build.mmd()
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['buildrequires']['platform']['stream'] = 'different'
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.rebuild_strategy = rebuild_strategy
db.session.commit()
plc_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-List-Compare')
pt_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'perl-Tangerine')
tangerine_rv = module_build_service.utils.get_reusable_component(
db.session, second_module_build, 'tangerine')
assert plc_rv is None
assert pt_rv is None
assert tangerine_rv is None
def test_get_reusable_component_different_buildrequires(self):
@@ -182,7 +215,7 @@ class TestUtilsComponentReuse:
}
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
db.session.commit()
plc_rv = module_build_service.utils.get_reusable_component(

3
tests/test_utils/test_utils_mse.py
View File

@@ -116,6 +116,7 @@ class TestUtilsModuleStreamExpansion:
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = 'changed-and-after'
module_build.build_context = context
module_build.stream_build_context = context
module_build.runtime_context = context
module_build.modulemd = mmd.dumps()
db.session.add(module_build)
@@ -160,7 +161,7 @@ class TestUtilsModuleStreamExpansion:
mmds = module_build_service.utils.generate_expanded_mmds(
db.session, module_build.mmd())
contexts = set([mmd.get_context() for mmd in mmds])
assert set(['3031e5a5', '6d10e00e']) == contexts
assert set(['ea432ace', 'b613fe68']) == contexts
@pytest.mark.parametrize(
'requires,build_requires,stream_ambigous,expected_xmd,expected_buildrequires', [