modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 11:48:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix #41 - Tag builds only in the end of the badge to not regen repository all the time.

Browse Source
This commit is contained in:
Jan Kaluza
2016-12-19 16:18:12 +01:00
parent 174e657379
commit ab567803e8
5 changed files with 113 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
module_build_service/builder.py
View File

@@ -212,6 +212,15 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
"""
raise NotImplementedError()
@abstractmethod
def tag_artifacts(self, artifacts):
"""
:param artifacts: list of artifacts (NVRs) to be tagged
Adds the artifacts to tag associated with this module build.
"""
raise NotImplementedError()
@abstractmethod
def build(self, artifact_name, source):
"""
@@ -282,6 +291,7 @@ class KojiModuleBuilder(GenericBuilder):
"""
self.owner = owner
self.module_str = module
self.config = config
self.tag_name = tag_name
self.__prep = False
log.debug("Using koji profile %r" % config.koji_profile)
@@ -482,11 +492,11 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
This method is safe to call multiple times.
"""
log.info("%r adding artifacts %r" % (self, artifacts))
dest_tag = self._get_tag(self.module_build_tag)['id']
build_tag = self._get_tag(self.module_build_tag)['id']
for nvr in artifacts:
log.info("%r tagging %r into %r" % (self, nvr, dest_tag))
self.koji_session.tagBuild(dest_tag, nvr, force=True)
log.info("%r tagging %r into %r" % (self, nvr, build_tag))
self.koji_session.tagBuild(build_tag, nvr, force=True)
if not install:
continue
@@ -494,7 +504,14 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
for group in ('srpm-build', 'build'):
name = kobo.rpmlib.parse_nvr(nvr)['name']
log.info("%r adding %s to group %s" % (self, name, group))
self.koji_session.groupPackageListAdd(dest_tag, group, name)
self.koji_session.groupPackageListAdd(build_tag, group, name)
def tag_artifacts(self, artifacts):
dest_tag = self._get_tag(self.module_tag)['id']
for nvr in artifacts:
log.info("%r tagging %r into %r" % (self, nvr, dest_tag))
self.koji_session.tagBuild(dest_tag, nvr, force=True)
def wait_task(self, task_id):
"""
@@ -580,7 +597,19 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
self.koji_session.uploadWrapper(source, serverdir, callback=callback)
source = "%s/%s" % (serverdir, os.path.basename(source))
task_id = self.koji_session.build(source, self.module_target['name'],
# When "koji_build_macros_target" is set, we build the
# module-build-macros in this target instead of the self.module_target.
# The reason is that it is faster to build this RPM in
# already existing shared target, because Koji does not need to do
# repo-regen.
if (artifact_name == "module-build-macros"
and self.config.koji_build_macros_target):
module_target = self.config.koji_build_macros_target
else:
module_target = self.module_target['name']
build_opts = {"skip_tag": True}
task_id = self.koji_session.build(source, module_target, build_opts,
priority=self.build_priority)
log.info("submitted build of %s (task_id=%s), via %s" % (
source, task_id, self))
@@ -874,6 +903,9 @@ class CoprModuleBuilder(GenericBuilder):
if backend == "copr":
return self._dependency_repo(module, arch, "koji")
def tag_artifacts(self, artifacts):
pass
def build(self, artifact_name, source):
"""
:param artifact_name : A package name. We can't guess it since macros
@@ -1153,6 +1185,9 @@ $repos
_execute_cmd(["mock", "-r", self.mock_config, "-i",
"module-build-macros"])
def tag_artifacts(self, artifacts):
pass
def buildroot_add_repos(self, dependencies):
# TODO: We support only dependencies from Koji here. This should be
# extended to Copr in the future.

5
module_build_service/config.py
View File

@@ -116,6 +116,11 @@ class Config(object):
'type': str,
'default': None,
'desc': 'Koji repository URL.'},
'koji_build_macros_target': {
'type': str,
'default': '',
'desc': 'Target to build "module-build-macros" RPM in.'
},
'rpms_default_repository': {
'type': str,
'default': 'git://pkgs.fedoraproject.org/rpms/',

3
module_build_service/scheduler/__init__.py
View File

@@ -9,7 +9,6 @@ import module_build_service.scheduler.consumer
import logging
log = logging.getLogger(__name__)
def main(initial_messages, stop_condition):
""" Run the consumer until some condition is met.
@@ -36,6 +35,8 @@ def main(initial_messages, stop_condition):
options=config,
# Only run the specified consumers if any are so specified.
consumers=consumers,
# Do not run default producers.
producers=[],
# Tell moksha to quiet its logging.
framework=False,
)

42
module_build_service/scheduler/handlers/components.py
View File

@@ -38,7 +38,7 @@ logging.basicConfig(level=logging.DEBUG)
def _finalize(config, session, msg, state):
""" Called whenever a koji build completes or fails. """
# First, find our ModuleBuild associated with this repo, if any.
# First, find our ModuleBuild associated with this component, if any.
component_build = models.ComponentBuild.from_component_event(session, msg)
try:
nvr = "{}-{}-{}".format(msg.build_name, msg.build_version,
@@ -63,18 +63,26 @@ def _finalize(config, session, msg, state):
parent = component_build.module_build
if component_build.package == 'module-build-macros':
if state != koji.BUILD_STATES['COMPLETE']:
# If the macro build failed, then the module is doomed.
parent.transition(config, state=models.BUILD_STATES['failed'],
state_reason=state_reason)
session.commit()
return
# If the macro build failed, then the module is doomed.
if (component_build.package == 'module-build-macros'
and state != koji.BUILD_STATES['COMPLETE']):
parent.transition(config, state=models.BUILD_STATES['failed'],
state_reason=state_reason)
session.commit()
return
# If there are no other components still building in a batch,
# we can tag all successfully built components in the batch.
unbuilt_components_in_batch = [
c for c in parent.current_batch()
if c.state == koji.BUILD_STATES['BUILDING'] or not c.state
]
if not unbuilt_components_in_batch:
built_components_in_batch = [
c.nvr for c in parent.current_batch()
if c.state == koji.BUILD_STATES['COMPLETE']
]
# TODO -- we should just do this with a koji target that feeds -build.
# Otherwise.. if it didn't fail, then tag it.
if state == koji.BUILD_STATES['COMPLETE']:
# And install the macros.
module_name = parent.name
tag = parent.koji_tag
builder = module_build_service.builder.GenericBuilder.create(
@@ -88,16 +96,16 @@ def _finalize(config, session, msg, state):
except ValueError:
reason = "Failed to gather buildroot groups from SCM."
log.exception(reason)
parent.transition(config, state="failed", state_reason=reason)
parent.transition(config, state=models.BUILD_STATES["failed"],
state_reason=reason)
session.commit()
raise
builder.buildroot_connect(groups)
# tag && add to srpm-build group
nvr = "{}-{}-{}".format(msg.build_name, msg.build_version,
msg.build_release)
# tag && add to srpm-build group if neccessary
install = bool(component_build.package == 'module-build-macros')
builder.buildroot_add_artifacts([nvr,], install=install)
builder.buildroot_add_artifacts(built_components_in_batch, install=install)
builder.tag_artifacts(built_components_in_batch)
session.commit()

46
tests/test_build/test_build.py
View File

@@ -78,6 +78,8 @@ class TestModuleBuilder(GenericBuilder):
on_build_cb = None
on_cancel_cb = None
on_buildroot_add_artifacts_cb = None
on_tag_artifacts_cb = None
def __init__(self, owner, module, config, tag_name):
self.module_str = module
@@ -90,6 +92,8 @@ class TestModuleBuilder(GenericBuilder):
TestModuleBuilder.INSTANT_COMPLETE = False
TestModuleBuilder.on_build_cb = None
TestModuleBuilder.on_cancel_cb = None
TestModuleBuilder.on_buildroot_add_artifacts_cb = None
TestModuleBuilder.on_tag_artifacts_cb = None
def buildroot_connect(self, groups):
pass
@@ -107,11 +111,16 @@ class TestModuleBuilder(GenericBuilder):
pass
def buildroot_add_artifacts(self, artifacts, install=False):
pass
if TestModuleBuilder.on_buildroot_add_artifacts_cb:
TestModuleBuilder.on_buildroot_add_artifacts_cb(self, artifacts, install)
def buildroot_add_repos(self, dependencies):
pass
def tag_artifacts(self, artifacts):
if TestModuleBuilder.on_tag_artifacts_cb:
TestModuleBuilder.on_tag_artifacts_cb(self, artifacts)
@property
def module_build_tag(self):
return {"name": self.tag_name + "-build"}
@@ -130,7 +139,7 @@ class TestModuleBuilder(GenericBuilder):
msg_id='a faked internal message',
build_id=build_id,
task_id=build_id,
build_name="name",
build_name=path.basename(source),
build_new_state=state,
build_release="1",
build_version="1"
@@ -211,7 +220,30 @@ class TestBuild(unittest.TestCase):
data = json.loads(rv.data)
module_build_id = data['id']
msgs = [RidaModule("fake msg", 1, 1)]
# Check that components are tagged after the batch is built.
tag_groups = []
tag_groups.append([u'module-build-macros-0.1-1.module_testmodule_teststream_1.src.rpm-1-1'])
tag_groups.append([u'perl-Tangerine?#f25-1-1', u'perl-List-Compare?#f25-1-1'])
tag_groups.append([u'tangerine?#f25-1-1'])
def on_tag_artifacts_cb(cls, artifacts):
self.assertEqual(tag_groups.pop(0), artifacts)
TestModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb
# Check that the components are added to buildroot after the batch
# is built.
buildroot_groups = []
buildroot_groups.append([u'module-build-macros-0.1-1.module_testmodule_teststream_1.src.rpm-1-1'])
buildroot_groups.append([u'perl-Tangerine?#f25-1-1', u'perl-List-Compare?#f25-1-1'])
buildroot_groups.append([u'tangerine?#f25-1-1'])
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
self.assertEqual(buildroot_groups.pop(0), artifacts)
TestModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb
msgs = []
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
module_build_service.scheduler.main(msgs, stop)
@@ -221,6 +253,10 @@ class TestBuild(unittest.TestCase):
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
# All components has to be tagged, so tag_groups and buildroot_groups are empty...
self.assertEqual(tag_groups, [])
self.assertEqual(buildroot_groups, [])
@timed(30)
@patch('module_build_service.auth.get_username', return_value='Homer J. Simpson')
@patch('module_build_service.auth.assert_is_packager')
@@ -257,7 +293,7 @@ class TestBuild(unittest.TestCase):
TestModuleBuilder.on_build_cb = on_build_cb
TestModuleBuilder.on_cancel_cb = on_cancel_cb
msgs = [RidaModule("fake msg", 1, 1)]
msgs = []
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
module_build_service.scheduler.main(msgs, stop)
@@ -296,7 +332,7 @@ class TestBuild(unittest.TestCase):
TestModuleBuilder.BUILD_STATE = "BUILDING"
TestModuleBuilder.INSTANT_COMPLETE = True
msgs = [RidaModule("fake msg", 1, 1)]
msgs = []
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
module_build_service.scheduler.main(msgs, stop)