From b0c4fbb11ca59c4238a9b832fa424572a2be9629 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:17:04 +0200 Subject: [PATCH 1/5] builder.py - fix dist-tag srpm creation, add option to pre-install artifact in buildroot --- rida/builder.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index adc34140..8c8e20c2 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -40,6 +40,7 @@ import datetime import time import random import string +import kobo.rpmlib import munch from OpenSSL.SSL import SysCallError @@ -123,9 +124,10 @@ class GenericBuilder: raise NotImplementedError() @abstractmethod - def buildroot_add_artifacts(self, artifacts): + def buildroot_add_artifacts(self, artifacts, install=False): """ :param artifacts: list of artifacts to be available in buildroot + :param install=False: pre-install artifact in buildroot (otherwise "make it available for install") add artifacts into buildroot, can be used to override buildroot macros """ raise NotImplementedError() @@ -161,13 +163,14 @@ class Builder: def retry(callback, **kwargs): attempt = 0 + log.debug("retry() calling %r(kwargs=%r)" % (callback, kwargs)) while True: try: callback(**kwargs) break except SysCallError: attempt += 1 - log.debug("Retry(): attempt=%d retrying callback." % attempt) + log.warn("retry(attempt=%d) calling %r(kwargs=%r)" % (attempt, callback, kwargs)) class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ @@ -259,9 +262,9 @@ It should NEVER be installed on any system as it will really mess up %build %install -mkdir -p %buildroot/%_rpmconfigdir/macro.d 2>/dev/null |: -echo %%dist %dist > %buildroot/%_rpmconfigdir/macro.modules -chmod 644 %buildroot/%_rpmconfigdir/macro.modules +mkdir -p %buildroot/%_rpmconfigdir/macros.d 2>/dev/null |: +echo %%dist %dist > %buildroot/%_rpmconfigdir/macros.d/macro.modules +chmod 644 %buildroot/%_rpmconfigdir/macros.d/macro.modules %files @@ -370,11 +373,23 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules log.info("%r adding deps for %r" % (self, tags)) self._koji_add_many_tag_inheritance(self.module_build_tag, tags) - def buildroot_add_artifacts(self, artifacts): + def buildroot_add_artifacts(self, artifacts, install=False): + """ + :param artifacts - list of artifacts to add to buildroot + :param install=False - force install artifact (if it's not dragged in as dependency) + """ # TODO: import /usr/bin/koji's TaskWatcher() log.info("%r adding artifacts %r" % (self, artifacts)) + dest_tag = self._get_tag(self.module_build_tag)['id'] for nvr in artifacts: - self.koji_session.tagBuild(self._get_tag(self.module_build_tag)['id'], nvr, force=True) + self.koji_session.tagBuild(dest_tag, nvr, force=True) + if install: + # we usually want just srpm-build + for group in ('srpm-build',): + pkg_info = kobo.rpmlib.parse_nvr(nvr) + log.info("%r adding %s to group %s" % (self, pkg_info['name'], group)) + self.koji_session.groupPackageListAdd(dest_tag, group, pkg_info['name']) + def wait_task(self, task_id): """ From 4ae2eb0bbc7ffb22aea85411876dfa4331eef945 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:18:03 +0200 Subject: [PATCH 2/5] set default dist-git and lookaside to stage --- rida.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida.conf b/rida.conf index dd332951..a50b61e9 100644 --- a/rida.conf +++ b/rida.conf @@ -19,9 +19,9 @@ port = 5000 # Set to zero to disable polling polling_interval = 60 -rpms_default_repository = git://pkgs.fedoraproject.org/rpms/ +rpms_default_repository = git://pkgs.stg.fedoraproject.org/rpms/ rpms_allow_repository = False -rpms_default_cache = http://pkgs.fedoraproject.org/repo/pkgs/ +rpms_default_cache = http://pkgs.stg.fedoraproject.org/repo/pkgs/ rpms_allow_cache = False ssl_enabled = True From 5ef69b02fb5a8f8aa5cb1e414d1af8e9c78710d1 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:18:21 +0200 Subject: [PATCH 3/5] pre-install macros in buildroot --- rida/scheduler/handlers/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 8e7a6aa3..246aead5 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -52,7 +52,6 @@ def wait(config, session, msg): module_info = build.json() if module_info['state'] != rida.BUILD_STATES["wait"]: # XXX: not sure why did we get here from state == 2 (building) FIXTHIS - print("Invalid state %s for wait()" % module_info['state']) log.error("Invalid state %s for wait(). Msg=%s" % (module_info['state'], msg)) return log.info("Found module_info=%s from message" % module_info) @@ -81,9 +80,10 @@ def wait(config, session, msg): # TODO -- this has to go eventually.. otherwise, we can only build one # module at a time and that just won't scale. builder.wait_task(task_id) + # TODO -- do cleanup if this fails artifact = get_artifact_from_srpm(srpm) - builder.buildroot_add_artifacts([artifact,]) # pretty much srpm filename + builder.buildroot_add_artifacts([artifact,], install=True) # tag && add to srpm-build group builder.buildroot_ready(artifacts=[artifact,]) build.transition(config, state="build") # Wait for the buildroot to be ready. From fd814ef0432c3b17c8df96177a4ac0a5ec6b9a87 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:18:45 +0200 Subject: [PATCH 4/5] add msg_id when we fake message --- rida/scheduler/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 8b28f7c6..9eb44e56 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -234,6 +234,7 @@ class Poller(threading.Thread): self.outgoing_work_queue.put({ 'topic': 'org.fedoraproject.prod.buildsys.build.state.change', 'msg': { + 'msg_id': 'a faked internal message', 'task_id': component_build.task_id, 'new': koji.BUILD_STATES['FAILED'], }, From 1bd9b28b92534b06431eefb04244d529b1f990b1 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:19:17 +0200 Subject: [PATCH 5/5] reflect bump of testmodule --- submit-build.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submit-build.json b/submit-build.json index b31328e9..8233168b 100644 --- a/submit-build.json +++ b/submit-build.json @@ -1,3 +1,3 @@ { - "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#5188d22b255c9f54798926959f43967a057ca690" + "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#d0c328b61a1b842bb5c400fc8eeb588af7746d2d" }