From 3b7108dc073d4cac686a8fc45d1c2f1ab92dc0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Wed, 5 Apr 2017 21:14:26 +0200 Subject: [PATCH 1/7] Install module-build-macros to the buildroot --- module_build_service/builder/CoprModuleBuilder.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 17293065..8e2ecaa0 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -159,6 +159,17 @@ class CoprModuleBuilder(GenericBuilder): koji add-group-pkg $module-build-tag srpm-build bash """ + # Install the module-build-macros into the buildroot + # We are using same hack as mock builder does + for artifact in artifacts: + if artifact and artifact.startswith("module-build-macros"): + chroot = self.client.get_chroot(self.copr.projectname, self.copr.username, "fedora-24-x86_64") + packages = (chroot.data["chroot"]["buildroot_pkgs"] or "").split() + self.client.edit_chroot(self.copr.projectname, "fedora-24-x86_64", + ownername=self.copr.username, + packages=" ".join(set(["module-build-macros"] + packages))) + break + # Start of a new batch of builds is triggered by buildsys.repo.done message. # However in Copr there is no such thing. Therefore we are going to fake # the message when builds are finished From 70185edad4aaeca6b3abe99de4b84338841d306c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Thu, 6 Apr 2017 18:02:11 +0200 Subject: [PATCH 2/7] Move chroot-updating logic to separate function --- module_build_service/builder/CoprModuleBuilder.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 8e2ecaa0..21dc922c 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -163,11 +163,7 @@ class CoprModuleBuilder(GenericBuilder): # We are using same hack as mock builder does for artifact in artifacts: if artifact and artifact.startswith("module-build-macros"): - chroot = self.client.get_chroot(self.copr.projectname, self.copr.username, "fedora-24-x86_64") - packages = (chroot.data["chroot"]["buildroot_pkgs"] or "").split() - self.client.edit_chroot(self.copr.projectname, "fedora-24-x86_64", - ownername=self.copr.username, - packages=" ".join(set(["module-build-macros"] + packages))) + self._update_chroot(packages=["module-build-macros"]) break # Start of a new batch of builds is triggered by buildsys.repo.done message. @@ -181,6 +177,13 @@ class CoprModuleBuilder(GenericBuilder): repos = [self._dependency_repo(d, "x86_64") for d in dependencies] self.client.modify_project(self.copr.projectname, username=self.copr.username, repos=repos) + def _update_chroot(self, packages=None): + chroot = self.client.get_chroot(self.copr.projectname, self.copr.username, "fedora-24-x86_64") + current_packages = (chroot.data["chroot"]["buildroot_pkgs"] or "").split() + self.client.edit_chroot(self.copr.projectname, "fedora-24-x86_64", + ownername=self.copr.username, + packages=" ".join(set(packages + current_packages))) + def _dependency_repo(self, module, arch, backend="copr"): try: repo = GenericBuilder.tag_to_repo(backend, self.config, module, arch) From 62d6930d927ed2771fad595c1fee68e904d53320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Thu, 6 Apr 2017 18:30:32 +0200 Subject: [PATCH 3/7] Update repos in the chroot instead of the repos for the whole project --- .../builder/CoprModuleBuilder.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 21dc922c..4c235fd8 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -175,14 +175,22 @@ class CoprModuleBuilder(GenericBuilder): log.info("%r adding deps on %r" % (self, dependencies)) # @TODO get architecture from some builder variable repos = [self._dependency_repo(d, "x86_64") for d in dependencies] - self.client.modify_project(self.copr.projectname, username=self.copr.username, repos=repos) + self._update_chroot(repos=repos) + + def _update_chroot(self, packages=None, repos=None): + request = self.client.get_chroot(self.copr.projectname, self.copr.username, "fedora-24-x86_64") + chroot = request.data["chroot"] + current_packages = (chroot["buildroot_pkgs"] or "").split() + current_repos = (chroot["repos"] or "").split() + + def merge(current, new): + current, new = current or [], new or [] + return " ".join(set(current + new)) - def _update_chroot(self, packages=None): - chroot = self.client.get_chroot(self.copr.projectname, self.copr.username, "fedora-24-x86_64") - current_packages = (chroot.data["chroot"]["buildroot_pkgs"] or "").split() self.client.edit_chroot(self.copr.projectname, "fedora-24-x86_64", ownername=self.copr.username, - packages=" ".join(set(packages + current_packages))) + packages=merge(current_packages, packages), + repos=merge(current_repos, repos)) def _dependency_repo(self, module, arch, backend="copr"): try: From 04270ebc7fdeac93ed9881a7f0eeedefb3933422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Thu, 6 Apr 2017 19:45:19 +0200 Subject: [PATCH 4/7] Install some required packages into the buildroot --- module_build_service/builder/CoprModuleBuilder.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 4c235fd8..017d3f74 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -54,6 +54,13 @@ class CoprModuleBuilder(GenericBuilder): backend = "copr" _build_lock = threading.Lock() + buildroot_packages = [ + "unzip", "tar", "cpio", "gawk", "xz", "sed", + "findutils", "util-linux", "bash", "info", "gcc", + "grep", "redhat-rpm-config", "shadow-utils", "rpm-build", + "rpm", "coreutils", "fedora-modular-release", "diffutils", + "make", "patch", "shadow-utils", + ] @module_build_service.utils.validate_koji_tag('tag_name') def __init__(self, owner, module, config, tag_name, components): @@ -81,6 +88,7 @@ class CoprModuleBuilder(GenericBuilder): """ self.copr = self._get_copr_safe() self._create_module_safe() + self._update_chroot(packages=self.buildroot_packages) if self.copr and self.copr.projectname and self.copr.username: self.__prep = True log.info("%r buildroot sucessfully connected." % self) From 28dda0a5c8ca47aefbac4b987ca862bdd85ba1b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Thu, 6 Apr 2017 22:44:09 +0200 Subject: [PATCH 5/7] Use empty chroot with base-runtime --- module_build_service/builder/CoprModuleBuilder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 017d3f74..11ed0f74 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -72,6 +72,7 @@ class CoprModuleBuilder(GenericBuilder): self.copr = None self.client = CoprModuleBuilder._get_client(config) self.client.username = self.owner + self.chroot = "custom-1-x86_64" self.__prep = False @classmethod @@ -112,8 +113,7 @@ class CoprModuleBuilder(GenericBuilder): return self.client.get_project_details(projectname, username=ownername).handle def _create_copr(self, ownername, projectname): - # @TODO fix issues with custom-1-x86_64 and custom-1-i386 chroot and use it - return self.client.create_project(ownername, projectname, ["fedora-24-x86_64"]) + return self.client.create_project(ownername, projectname, [self.chroot]) def _create_module_safe(self): from copr.exceptions import CoprRequestException @@ -186,7 +186,7 @@ class CoprModuleBuilder(GenericBuilder): self._update_chroot(repos=repos) def _update_chroot(self, packages=None, repos=None): - request = self.client.get_chroot(self.copr.projectname, self.copr.username, "fedora-24-x86_64") + request = self.client.get_chroot(self.copr.projectname, self.copr.username, self.chroot) chroot = request.data["chroot"] current_packages = (chroot["buildroot_pkgs"] or "").split() current_repos = (chroot["repos"] or "").split() @@ -195,7 +195,7 @@ class CoprModuleBuilder(GenericBuilder): current, new = current or [], new or [] return " ".join(set(current + new)) - self.client.edit_chroot(self.copr.projectname, "fedora-24-x86_64", + self.client.edit_chroot(self.copr.projectname, self.chroot, ownername=self.copr.username, packages=merge(current_packages, packages), repos=merge(current_repos, repos)) From 14ee3a45918ace5fd912ae0099df423dfe7f176e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 10 Apr 2017 15:56:02 +0200 Subject: [PATCH 6/7] Don't manage buildroot packages manually --- module_build_service/builder/CoprModuleBuilder.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 11ed0f74..e12d1670 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -54,13 +54,6 @@ class CoprModuleBuilder(GenericBuilder): backend = "copr" _build_lock = threading.Lock() - buildroot_packages = [ - "unzip", "tar", "cpio", "gawk", "xz", "sed", - "findutils", "util-linux", "bash", "info", "gcc", - "grep", "redhat-rpm-config", "shadow-utils", "rpm-build", - "rpm", "coreutils", "fedora-modular-release", "diffutils", - "make", "patch", "shadow-utils", - ] @module_build_service.utils.validate_koji_tag('tag_name') def __init__(self, owner, module, config, tag_name, components): @@ -75,6 +68,7 @@ class CoprModuleBuilder(GenericBuilder): self.chroot = "custom-1-x86_64" self.__prep = False + @classmethod def _get_client(cls, config): return CoprClient.create_from_file_config(config.copr_config) @@ -89,7 +83,11 @@ class CoprModuleBuilder(GenericBuilder): """ self.copr = self._get_copr_safe() self._create_module_safe() - self._update_chroot(packages=self.buildroot_packages) + + # @FIXME Not able to use gcc-c++ in chroot (RhBug: 1440889) + packages = groups["build"] - {"gcc-c++"} + self._update_chroot(packages=list(packages)) + if self.copr and self.copr.projectname and self.copr.username: self.__prep = True log.info("%r buildroot sucessfully connected." % self) From 7c81a72138caa95e2abe1295b005c7367bf21136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 10 Apr 2017 20:28:53 +0200 Subject: [PATCH 7/7] Update CoprBuildEnd message to reflect actual data --- module_build_service/messaging.py | 14 +++++++------- tests/test_messaging.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index 52b32ba6..2d9c5e05 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -27,7 +27,6 @@ import json import os import re -import kobo.rpmlib try: from inspect import signature except ImportError: @@ -225,8 +224,9 @@ class BaseMessage(object): build = msg_inner_msg.get('build') status = msg_inner_msg.get('status') pkg = msg_inner_msg.get('pkg') + version = msg_inner_msg.get('version') what = msg_inner_msg.get('what') - msg_obj = CoprBuildEnd(msg_id, build, status, copr, pkg, what) + msg_obj = CoprBuildEnd(msg_id, build, status, copr, pkg, version, what) # If the message matched the regex and is important to the app, # it will be returned @@ -305,16 +305,16 @@ class CoprBuildEnd(KojiBuildChange): (e.g. mutt-kz-1.5.23.1-1.20150203.git.c8504a8a.fc21) :param state_reason: the optional reason as to why the state changed """ - def __init__(self, msg_id, build_id, status, copr, pkg, what=None): - nvr = kobo.rpmlib.parse_nvra(pkg) + def __init__(self, msg_id, build_id, status, copr, pkg, version, what=None): + ver, rel = version.split("-", 1) super(CoprBuildEnd, self).__init__( msg_id=msg_id, build_id=build_id, task_id=build_id, build_new_state=status, - build_name=nvr["name"], - build_version=nvr["version"], - build_release=".".join(s for s in [nvr["release"], nvr["epoch"], nvr["arch"]] if s), + build_name=pkg, + build_version=ver, + build_release=rel, state_reason=what, ) self.copr = copr diff --git a/tests/test_messaging.py b/tests/test_messaging.py index 3e408a68..7af40b9a 100644 --- a/tests/test_messaging.py +++ b/tests/test_messaging.py @@ -65,7 +65,7 @@ class TestFedmsgMessaging(unittest.TestCase): 'copr': 'mutt-kz', 'ip': '172.16.3.3', 'pid': 12010, - 'pkg': 'mutt-kz-1.5.23.1-1.20150203.git.c8504a8a.fc21', + 'pkg': 'mutt-kz', # Reality doesnt match the linked docs 'status': 1, 'user': 'fatka', 'version': '1.5.23.1-1.20150203.git.c8504a8a.fc21',