modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 19:57:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #274 Fix #41 - Tag builds only in the end of the batch to not regen repository all the time.

Browse Source
This commit is contained in:
Jan Kaluža
2017-01-06 15:27:08 +00:00
parent b28a5a36aa 23802418af
commit 87830b5b13
5 changed files with 187 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
module_build_service/builder.py
View File

@@ -212,6 +212,15 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
"""
raise NotImplementedError()
@abstractmethod
def tag_artifacts(self, artifacts):
"""
:param artifacts: list of artifacts (NVRs) to be tagged
Adds the artifacts to tag associated with this module build.
"""
raise NotImplementedError()
@abstractmethod
def build(self, artifact_name, source):
"""
@@ -282,6 +291,7 @@ class KojiModuleBuilder(GenericBuilder):
"""
self.owner = owner
self.module_str = module
self.config = config
self.tag_name = tag_name
self.__prep = False
log.debug("Using koji profile %r" % config.koji_profile)
@@ -482,11 +492,11 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
This method is safe to call multiple times.
"""
log.info("%r adding artifacts %r" % (self, artifacts))
dest_tag = self._get_tag(self.module_build_tag)['id']
build_tag = self._get_tag(self.module_build_tag)['id']
for nvr in artifacts:
log.info("%r tagging %r into %r" % (self, nvr, dest_tag))
self.koji_session.tagBuild(dest_tag, nvr, force=True)
log.info("%r tagging %r into %r" % (self, nvr, build_tag))
self.koji_session.tagBuild(build_tag, nvr, force=True)
if not install:
continue
@@ -494,7 +504,14 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
for group in ('srpm-build', 'build'):
name = kobo.rpmlib.parse_nvr(nvr)['name']
log.info("%r adding %s to group %s" % (self, name, group))
self.koji_session.groupPackageListAdd(dest_tag, group, name)
self.koji_session.groupPackageListAdd(build_tag, group, name)
def tag_artifacts(self, artifacts):
dest_tag = self._get_tag(self.module_tag)['id']
for nvr in artifacts:
log.info("%r tagging %r into %r" % (self, nvr, dest_tag))
self.koji_session.tagBuild(dest_tag, nvr, force=True)
def wait_task(self, task_id):
"""
@@ -580,7 +597,19 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
self.koji_session.uploadWrapper(source, serverdir, callback=callback)
source = "%s/%s" % (serverdir, os.path.basename(source))
task_id = self.koji_session.build(source, self.module_target['name'],
# When "koji_build_macros_target" is set, we build the
# module-build-macros in this target instead of the self.module_target.
# The reason is that it is faster to build this RPM in
# already existing shared target, because Koji does not need to do
# repo-regen.
if (artifact_name == "module-build-macros"
and self.config.koji_build_macros_target):
module_target = self.config.koji_build_macros_target
else:
module_target = self.module_target['name']
build_opts = {"skip_tag": True}
task_id = self.koji_session.build(source, module_target, build_opts,
priority=self.build_priority)
log.info("submitted build of %s (task_id=%s), via %s" % (
source, task_id, self))
@@ -874,6 +903,9 @@ class CoprModuleBuilder(GenericBuilder):
if backend == "copr":
return self._dependency_repo(module, arch, "koji")
def tag_artifacts(self, artifacts):
pass
def build(self, artifact_name, source):
"""
:param artifact_name : A package name. We can't guess it since macros
@@ -1153,6 +1185,16 @@ $repos
_execute_cmd(["mock", "-r", self.mock_config, "-i",
"module-build-macros"])
def _send_repo_done(self):
msg = module_build_service.messaging.KojiRepoChange(
msg_id='a faked internal message',
repo_tag=self.tag_name + "-build",
)
module_build_service.scheduler.consumer.work_queue_put(msg)
def tag_artifacts(self, artifacts):
self._send_repo_done()
def buildroot_add_repos(self, dependencies):
# TODO: We support only dependencies from Koji here. This should be
# extended to Copr in the future.
@@ -1161,13 +1203,6 @@ $repos
self._add_repo(tag, baseurl)
self._write_mock_config()
def _send_repo_done(self):
msg = module_build_service.messaging.KojiRepoChange(
msg_id='a faked internal message',
repo_tag=self.tag_name + "-build",
)
module_build_service.scheduler.consumer.work_queue_put(msg)
def _send_build_change(self, state, source, build_id):
nvr = kobo.rpmlib.parse_nvr(source)
@@ -1209,10 +1244,8 @@ $repos
# are put in the scheduler's work queue and are handled
# by MBS after the build_srpm() method returns and scope gets
# back to scheduler.main.main() method.
self._send_repo_done()
self._send_build_change(koji.BUILD_STATES['COMPLETE'], source,
MockModuleBuilder._build_id)
self._send_repo_done()
with open(os.path.join(self.resultsdir, "status.log"), 'w') as f:
f.write("complete\n")
@@ -1224,10 +1257,8 @@ $repos
# are put in the scheduler's work queue and are handled
# by MBS after the build_srpm() method returns and scope gets
# back to scheduler.main.main() method.
self._send_repo_done()
self._send_build_change(koji.BUILD_STATES['FAILED'], source,
MockModuleBuilder._build_id)
self._send_repo_done()
with open(os.path.join(self.resultsdir, "status.log"), 'w') as f:
f.write("failed\n")

5
module_build_service/config.py
View File

@@ -116,6 +116,11 @@ class Config(object):
'type': str,
'default': None,
'desc': 'Koji repository URL.'},
'koji_build_macros_target': {
'type': str,
'default': '',
'desc': 'Target to build "module-build-macros" RPM in.'
},
'rpms_default_repository': {
'type': str,
'default': 'git://pkgs.fedoraproject.org/rpms/',

3
module_build_service/scheduler/__init__.py
View File

@@ -9,7 +9,6 @@ import module_build_service.scheduler.consumer
import logging
log = logging.getLogger(__name__)
def main(initial_messages, stop_condition):
""" Run the consumer until some condition is met.
@@ -36,6 +35,8 @@ def main(initial_messages, stop_condition):
options=config,
# Only run the specified consumers if any are so specified.
consumers=consumers,
# Do not run default producers.
producers=[],
# Tell moksha to quiet its logging.
framework=False,
)

91
module_build_service/scheduler/handlers/components.py
View File

@@ -38,7 +38,7 @@ logging.basicConfig(level=logging.DEBUG)
def _finalize(config, session, msg, state):
""" Called whenever a koji build completes or fails. """
# First, find our ModuleBuild associated with this repo, if any.
# First, find our ModuleBuild associated with this component, if any.
component_build = models.ComponentBuild.from_component_event(session, msg)
try:
nvr = "{}-{}-{}".format(msg.build_name, msg.build_version,
@@ -63,42 +63,61 @@ def _finalize(config, session, msg, state):
parent = component_build.module_build
if component_build.package == 'module-build-macros':
if state != koji.BUILD_STATES['COMPLETE']:
# If the macro build failed, then the module is doomed.
parent.transition(config, state=models.BUILD_STATES['failed'],
state_reason=state_reason)
session.commit()
return
# TODO -- we should just do this with a koji target that feeds -build.
# Otherwise.. if it didn't fail, then tag it.
if state == koji.BUILD_STATES['COMPLETE']:
# And install the macros.
module_name = parent.name
tag = parent.koji_tag
builder = module_build_service.builder.GenericBuilder.create(
parent.owner, module_name, config.system, config, tag_name=tag)
try:
groups = {
'build': parent.resolve_profiles(session, 'buildroot'),
'srpm-build': parent.resolve_profiles(session, 'srpm-buildroot'),
}
except ValueError:
reason = "Failed to gather buildroot groups from SCM."
log.exception(reason)
parent.transition(config, state="failed", state_reason=reason)
session.commit()
raise
builder.buildroot_connect(groups)
# tag && add to srpm-build group
nvr = "{}-{}-{}".format(msg.build_name, msg.build_version,
msg.build_release)
install = bool(component_build.package == 'module-build-macros')
builder.buildroot_add_artifacts([nvr,], install=install)
# If the macro build failed, then the module is doomed.
if (component_build.package == 'module-build-macros'
and state != koji.BUILD_STATES['COMPLETE']):
parent.transition(config, state=models.BUILD_STATES['failed'],
state_reason=state_reason)
session.commit()
return
# Initialize the builder, we will need it later.
module_name = parent.name
tag = parent.koji_tag
builder = module_build_service.builder.GenericBuilder.create(
parent.owner, module_name, config.system, config, tag_name=tag)
try:
groups = {
'build': parent.resolve_profiles(session, 'buildroot'),
'srpm-build': parent.resolve_profiles(session, 'srpm-buildroot'),
}
except ValueError:
reason = "Failed to gather buildroot groups from SCM."
log.exception(reason)
parent.transition(config, state=models.BUILD_STATES["failed"],
state_reason=reason)
session.commit()
raise
builder.buildroot_connect(groups)
# If there are no other components still building in a batch,
# we can tag all successfully built components in the batch.
unbuilt_components_in_batch = [
c for c in parent.current_batch()
if c.state == koji.BUILD_STATES['BUILDING'] or not c.state
]
if not unbuilt_components_in_batch:
built_components_in_batch = [
c.nvr for c in parent.current_batch()
if c.state == koji.BUILD_STATES['COMPLETE']
]
# tag && add to srpm-build group if neccessary
install = bool(component_build.package == 'module-build-macros')
builder.buildroot_add_artifacts(built_components_in_batch, install=install)
builder.tag_artifacts(built_components_in_batch)
session.commit()
else:
# We have some unbuilt components in this batch. We might hit the
# concurrent builds threshold in previous call of start_build_batch
# done in repos.py:done(...), but because we have just finished one
# build, try to call start_build_batch again so in case we hit the
# threshold previously, we will submit another build from this batch.
further_work = module_build_service.utils.start_build_batch(
config, parent, session, builder)
return further_work
def complete(config, session, msg):

86
tests/test_build/test_build.py
View File

@@ -78,6 +78,8 @@ class TestModuleBuilder(GenericBuilder):
on_build_cb = None
on_cancel_cb = None
on_buildroot_add_artifacts_cb = None
on_tag_artifacts_cb = None
def __init__(self, owner, module, config, tag_name):
self.module_str = module
@@ -90,6 +92,8 @@ class TestModuleBuilder(GenericBuilder):
TestModuleBuilder.INSTANT_COMPLETE = False
TestModuleBuilder.on_build_cb = None
TestModuleBuilder.on_cancel_cb = None
TestModuleBuilder.on_buildroot_add_artifacts_cb = None
TestModuleBuilder.on_tag_artifacts_cb = None
def buildroot_connect(self, groups):
pass
@@ -107,11 +111,17 @@ class TestModuleBuilder(GenericBuilder):
pass
def buildroot_add_artifacts(self, artifacts, install=False):
pass
if TestModuleBuilder.on_buildroot_add_artifacts_cb:
TestModuleBuilder.on_buildroot_add_artifacts_cb(self, artifacts, install)
def buildroot_add_repos(self, dependencies):
pass
def tag_artifacts(self, artifacts):
if TestModuleBuilder.on_tag_artifacts_cb:
TestModuleBuilder.on_tag_artifacts_cb(self, artifacts)
self._send_repo_done()
@property
def module_build_tag(self):
return {"name": self.tag_name + "-build"}
@@ -130,7 +140,7 @@ class TestModuleBuilder(GenericBuilder):
msg_id='a faked internal message',
build_id=build_id,
task_id=build_id,
build_name="name",
build_name=path.basename(source),
build_new_state=state,
build_release="1",
build_version="1"
@@ -143,11 +153,9 @@ class TestModuleBuilder(GenericBuilder):
TestModuleBuilder._build_id += 1
if TestModuleBuilder.BUILD_STATE != "BUILDING":
self._send_repo_done()
self._send_build_change(
koji.BUILD_STATES[TestModuleBuilder.BUILD_STATE], source,
TestModuleBuilder._build_id)
self._send_repo_done()
if TestModuleBuilder.on_build_cb:
TestModuleBuilder.on_build_cb(self, artifact_name, source)
@@ -175,6 +183,9 @@ class TestBuild(unittest.TestCase):
def setUp(self):
GenericBuilder.register_backend_class(TestModuleBuilder)
self.client = app.test_client()
self._prev_system = conf.system
self._prev_num_consencutive_builds = conf.num_consecutive_builds
conf.set_item("system", "mock")
init_data()
@@ -182,7 +193,8 @@ class TestBuild(unittest.TestCase):
models.ComponentBuild.query.delete()
def tearDown(self):
conf.set_item("system", "koji")
conf.set_item("system", self._prev_system)
conf.set_item("num_consecutive_builds", self._prev_num_consencutive_builds)
TestModuleBuilder.reset()
# Necessary to restart the twisted reactor for the next test.
@@ -211,7 +223,30 @@ class TestBuild(unittest.TestCase):
data = json.loads(rv.data)
module_build_id = data['id']
msgs = [RidaModule("fake msg", 1, 1)]
# Check that components are tagged after the batch is built.
tag_groups = []
tag_groups.append([u'module-build-macros-0.1-1.module_testmodule_teststream_1.src.rpm-1-1'])
tag_groups.append([u'perl-Tangerine?#f25-1-1', u'perl-List-Compare?#f25-1-1'])
tag_groups.append([u'tangerine?#f25-1-1'])
def on_tag_artifacts_cb(cls, artifacts):
self.assertEqual(tag_groups.pop(0), artifacts)
TestModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb
# Check that the components are added to buildroot after the batch
# is built.
buildroot_groups = []
buildroot_groups.append([u'module-build-macros-0.1-1.module_testmodule_teststream_1.src.rpm-1-1'])
buildroot_groups.append([u'perl-Tangerine?#f25-1-1', u'perl-List-Compare?#f25-1-1'])
buildroot_groups.append([u'tangerine?#f25-1-1'])
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
self.assertEqual(buildroot_groups.pop(0), artifacts)
TestModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb
msgs = []
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
module_build_service.scheduler.main(msgs, stop)
@@ -221,6 +256,10 @@ class TestBuild(unittest.TestCase):
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
# All components has to be tagged, so tag_groups and buildroot_groups are empty...
self.assertEqual(tag_groups, [])
self.assertEqual(buildroot_groups, [])
@timed(30)
@patch('module_build_service.auth.get_username', return_value='Homer J. Simpson')
@patch('module_build_service.auth.assert_is_packager')
@@ -257,7 +296,7 @@ class TestBuild(unittest.TestCase):
TestModuleBuilder.on_build_cb = on_build_cb
TestModuleBuilder.on_cancel_cb = on_cancel_cb
msgs = [RidaModule("fake msg", 1, 1)]
msgs = []
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
module_build_service.scheduler.main(msgs, stop)
@@ -296,7 +335,38 @@ class TestBuild(unittest.TestCase):
TestModuleBuilder.BUILD_STATE = "BUILDING"
TestModuleBuilder.INSTANT_COMPLETE = True
msgs = [RidaModule("fake msg", 1, 1)]
msgs = []
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
module_build_service.scheduler.main(msgs, stop)
# All components should be built and module itself should be in "done"
# or "ready" state.
for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
@timed(30)
@patch('module_build_service.auth.get_username', return_value='Homer J. Simpson')
@patch('module_build_service.auth.assert_is_packager')
@patch('module_build_service.scm.SCM')
def test_submit_build_concurrent_threshold(
self, mocked_scm, mocked_assert_is_packager, mocked_get_username):
"""
Tests the build of testmodule.yaml using TestModuleBuilder with
num_consecutive_builds set to 1.
"""
MockedSCM(mocked_scm, "testmodule", "testmodule.yaml")
conf.set_item("num_consecutive_builds", 1)
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
module_build_id = data['id']
msgs = []
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
module_build_service.scheduler.main(msgs, stop)