From 92102a2ac024ab7a104ad090beeff682de7e9dd6 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Mon, 24 Jul 2017 23:07:59 +0200 Subject: [PATCH 01/10] Don't build from scm locally --- module_build_service/builder/CoprModuleBuilder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index ccbdce7b..b43f62e7 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -241,7 +241,7 @@ class CoprModuleBuilder(GenericBuilder): with CoprModuleBuilder._build_lock: # Git sources are treated specially. if source.startswith(("git://", "http://", "https://")): - return build_from_scm(artifact_name, source, self.config, self.build_srpm) + return self.build_scm(source) else: return self.build_srpm(artifact_name, source) @@ -257,13 +257,14 @@ class CoprModuleBuilder(GenericBuilder): return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None def build_scm(self, source): - # @TODO use this method once support on Copr side is finished - # Copr is currently able to create a build from fedora distgit, - # but not from custom distgit, such as copr-dist-git url, branch = source.split("?#") url = (url.replace("git://", "https://") .replace("pkgs.fedoraproject.org", "src.fedoraproject.org/git")) - self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username) + response = self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username) + if response.output != "ok": + log.error(response.error) + + return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None def finalize(self): modulemd = tempfile.mktemp() From 33010a83bd638d42e7892bf9170f5b823744b6fd Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Mon, 24 Jul 2017 23:10:12 +0200 Subject: [PATCH 02/10] Refactor build methods to remove duplicity --- .../builder/CoprModuleBuilder.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index b43f62e7..d0508674 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -241,30 +241,26 @@ class CoprModuleBuilder(GenericBuilder): with CoprModuleBuilder._build_lock: # Git sources are treated specially. if source.startswith(("git://", "http://", "https://")): - return self.build_scm(source) + response = self.build_scm(source) else: - return self.build_srpm(artifact_name, source) + response = self.build_srpm(artifact_name, source) + + if response.output != "ok": + log.error(response.error) + return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None def build_srpm(self, artifact_name, source, build_id=None): if not self.__prep: raise RuntimeError("Buildroot is not prep-ed") # Build package from `source` - response = self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username) - if response.output != "ok": - log.error(response.error) - - return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None + return self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username) def build_scm(self, source): url, branch = source.split("?#") url = (url.replace("git://", "https://") .replace("pkgs.fedoraproject.org", "src.fedoraproject.org/git")) - response = self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username) - if response.output != "ok": - log.error(response.error) - - return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None + return self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username) def finalize(self): modulemd = tempfile.mktemp() From 6e6de42d0dcd8aac5c55a8cffa2fb8fccd18db0c Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Mon, 24 Jul 2017 23:30:24 +0200 Subject: [PATCH 03/10] Send the actual branch to copr, not commit --- .../builder/CoprModuleBuilder.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index d0508674..57018c46 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -257,9 +257,12 @@ class CoprModuleBuilder(GenericBuilder): return self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username) def build_scm(self, source): - url, branch = source.split("?#") + url, commit = source.split("?#") url = (url.replace("git://", "https://") .replace("pkgs.fedoraproject.org", "src.fedoraproject.org/git")) + cod = clone(url) + branch = git_branch_contains(cod, commit) + rmdir(cod) return self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username) def finalize(self): @@ -382,6 +385,23 @@ def build_from_scm(artifact_name, source, config, build_srpm, return ret +def clone(url): + log.debug('Cloning source URL: %s' % url) + td = tempfile.mkdtemp() + scm = module_build_service.scm.SCM(url) + return scm.checkout(td) + + +def rmdir(path): + try: + if path is not None: + shutil.rmtree(path) + except Exception as e: + log.warning( + "Failed to remove temporary directory {!r}: {}".format( + path, str(e))) + + def git_branch_contains(cod, commit): cmd = ["git", "branch", "-r", "--contains", commit] out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 9b1d1df0563f40c32b0cd10ee1a26cf880262997 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Mon, 24 Jul 2017 23:32:08 +0200 Subject: [PATCH 04/10] Remove the hacky build_from_scm code --- .../builder/CoprModuleBuilder.py | 69 ------------------- 1 file changed, 69 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 57018c46..901a5cff 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -325,66 +325,6 @@ class CoprModuleBuilder(GenericBuilder): return koji_tag.replace("+", "-") -def build_from_scm(artifact_name, source, config, build_srpm, - data=None, stdout=None, stderr=None): - """ - Builds the artifact from the SCM based source. - - :param artifact_name: Name of the artifact. - :param source: SCM URL with artifact's sources (spec file). - :param config: Config instance. - :param build_srpm: Method to call to build the RPM from the generate SRPM. - :param data: Data to be passed to the build_srpm method. - :param stdout: Python file object to which the stdout of SRPM build - command is logged. - :param stderr: Python file object to which the stderr of SRPM build - command is logged. - """ - ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM", None) - td = None - - try: - log.debug('Cloning source URL: %s' % source) - url, commit = source.split("?#") - # Create temp dir and clone the repo there. - td = tempfile.mkdtemp() - scm = module_build_service.scm.SCM(source) - cod = scm.checkout(td) - - cmd = config.mock_build_srpm_cmd.split(" ") - if is_from_copr(source): - branch = git_branch_contains(cod, commit) - if branch != "HEAD": - git_checkout(cod, branch) - cmd = ["fedpkg-copr", "--release", branch, "srpm"] - - # Use configured command to create SRPM out of the SCM repo. - log.debug("Creating SRPM in %s" % cod) - execute_cmd(cmd, stdout=stdout, stderr=stderr, cwd=cod) - - # Find out the built SRPM and build it normally. - for f in os.listdir(cod): - if f.endswith(".src.rpm"): - log.info("Created SRPM %s" % f) - source = os.path.join(cod, f) - ret = build_srpm(artifact_name, source, data) - break - except Exception as e: - log.error("Error while generating SRPM for artifact %s: %s" % ( - artifact_name, str(e))) - ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM %s" % str(e), None) - finally: - try: - if td is not None: - shutil.rmtree(td) - except Exception as e: - log.warning( - "Failed to remove temporary directory {!r}: {}".format( - td, str(e))) - - return ret - - def clone(url): log.debug('Cloning source URL: %s' % url) td = tempfile.mkdtemp() @@ -409,12 +349,3 @@ def git_branch_contains(cod, commit): if " -> " in branch: branch = branch.split(" -> ")[0] return branch - - -def is_from_copr(source): - return bool(re.match("https?://copr-dist-git(-dev)?\.fedorainfracloud\.org", source)) - - -def git_checkout(cod, branch): - cmd = ["git", "checkout", branch] - execute_cmd(cmd, cwd=cod) From ddd371d76cd32edcdb62f5f1e3991718c44a7c65 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Mon, 24 Jul 2017 23:33:43 +0200 Subject: [PATCH 05/10] Check whether the buildroot is preped for all build methods --- module_build_service/builder/CoprModuleBuilder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 901a5cff..92ff1fde 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -236,6 +236,9 @@ class CoprModuleBuilder(GenericBuilder): """ log.info("Copr build") + if not self.__prep: + raise RuntimeError("Buildroot is not prep-ed") + # TODO: If we are sure that this method is thread-safe, we can just # remove _build_lock locking. with CoprModuleBuilder._build_lock: @@ -250,9 +253,6 @@ class CoprModuleBuilder(GenericBuilder): return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None def build_srpm(self, artifact_name, source, build_id=None): - if not self.__prep: - raise RuntimeError("Buildroot is not prep-ed") - # Build package from `source` return self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username) From a84450007802d51c1119b95299d83fea20f1eb7e Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Fri, 28 Jul 2017 01:24:04 +0200 Subject: [PATCH 06/10] Build only in custom chroot --- module_build_service/builder/CoprModuleBuilder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 92ff1fde..746ddde9 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -254,7 +254,8 @@ class CoprModuleBuilder(GenericBuilder): def build_srpm(self, artifact_name, source, build_id=None): # Build package from `source` - return self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username) + return self.client.create_new_build(self.copr.projectname, [source],username=self.copr.username, + chroots=[self.chroot]) def build_scm(self, source): url, commit = source.split("?#") @@ -263,7 +264,8 @@ class CoprModuleBuilder(GenericBuilder): cod = clone(url) branch = git_branch_contains(cod, commit) rmdir(cod) - return self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username) + return self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, + username=self.copr.username, chroots=[self.chroot]) def finalize(self): modulemd = tempfile.mktemp() From bbcbb97c4595789025d4046af6c34487cfc2d86d Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Fri, 28 Jul 2017 21:13:55 +0200 Subject: [PATCH 07/10] There should be a space in order to be PEP-8 compliant --- module_build_service/builder/CoprModuleBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 746ddde9..30275090 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -254,7 +254,7 @@ class CoprModuleBuilder(GenericBuilder): def build_srpm(self, artifact_name, source, build_id=None): # Build package from `source` - return self.client.create_new_build(self.copr.projectname, [source],username=self.copr.username, + return self.client.create_new_build(self.copr.projectname, [source], username=self.copr.username, chroots=[self.chroot]) def build_scm(self, source): From 1c7cb20430e6ded4159a7a45629e2e22e6502ac6 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Tue, 1 Aug 2017 14:51:38 +0200 Subject: [PATCH 08/10] Properly clean temporary directories --- module_build_service/builder/CoprModuleBuilder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 30275090..b841a269 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -261,7 +261,8 @@ class CoprModuleBuilder(GenericBuilder): url, commit = source.split("?#") url = (url.replace("git://", "https://") .replace("pkgs.fedoraproject.org", "src.fedoraproject.org/git")) - cod = clone(url) + td = tempfile.mkdtemp() + cod = clone(url, td) branch = git_branch_contains(cod, commit) rmdir(cod) return self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, @@ -327,11 +328,10 @@ class CoprModuleBuilder(GenericBuilder): return koji_tag.replace("+", "-") -def clone(url): +def clone(url, path): log.debug('Cloning source URL: %s' % url) - td = tempfile.mkdtemp() scm = module_build_service.scm.SCM(url) - return scm.checkout(td) + return scm.checkout(path) def rmdir(path): From 23ce12a0995314669db6a2fa899c0af65d6538b3 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Thu, 3 Aug 2017 20:38:59 +0200 Subject: [PATCH 09/10] Sort branches from the oldest --- module_build_service/builder/CoprModuleBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index b841a269..d10c058e 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -345,7 +345,7 @@ def rmdir(path): def git_branch_contains(cod, commit): - cmd = ["git", "branch", "-r", "--contains", commit] + cmd = ["git", "branch", "-r", "--contains", commit, "--sort", "-committerdate"] out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE) branch = out.strip().split("/")[1] if " -> " in branch: From 457664184cdcea01d996f3a9ff689f937e3a16a9 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Thu, 3 Aug 2017 23:08:34 +0200 Subject: [PATCH 10/10] Handle multiline output better --- module_build_service/builder/CoprModuleBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index d10c058e..b1cd935f 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -347,7 +347,7 @@ def rmdir(path): def git_branch_contains(cod, commit): cmd = ["git", "branch", "-r", "--contains", commit, "--sort", "-committerdate"] out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - branch = out.strip().split("/")[1] + branch = out.split()[0].split("/")[1] if " -> " in branch: branch = branch.split(" -> ")[0] return branch