From 53c00dc0745dbbccc94726fe3373325212e24ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 29 May 2017 23:26:42 +0200 Subject: [PATCH 1/8] Allow stdout and stderr to be something else than file --- module_build_service/builder/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index 31633f13..6b2e163d 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -90,9 +90,9 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None): :raises RuntimeError: Raised when command exits with non-zero exit code. """ out_log_msg = "" - if stdout: + if stdout and hasattr(stdout, "name"): out_log_msg += ", stdout log: %s" % stdout.name - if stderr: + if stderr and hasattr(stderr, "name"): out_log_msg += ", stderr log: %s" % stderr.name log.info("Executing command: %s%s" % (args, out_log_msg)) From aee7c0b29919a62bbd63d1ccb4a9b34726d39b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 29 May 2017 23:28:00 +0200 Subject: [PATCH 2/8] Return proc.communicate() output --- module_build_service/builder/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index 6b2e163d..c9ec9a81 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -97,11 +97,12 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None): log.info("Executing command: %s%s" % (args, out_log_msg)) proc = subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd) - proc.communicate() + out, err = proc.communicate() if proc.returncode != 0: err_msg = "Command '%s' returned non-zero value %d%s" % (args, proc.returncode, out_log_msg) raise RuntimeError(err_msg) + return out, err def fake_repo_done_message(tag_name): From 91e08d723f0472ef6ae31c26a4dbb4f53a77714a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 29 May 2017 23:32:51 +0200 Subject: [PATCH 3/8] Allow build_from_scm to build even from custom distgits --- module_build_service/builder/utils.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index c9ec9a81..bd0b8bbc 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -37,14 +37,19 @@ def build_from_scm(artifact_name, source, config, build_srpm, try: log.debug('Cloning source URL: %s' % source) + url, commit = source.split("?#") # Create temp dir and clone the repo there. td = tempfile.mkdtemp() - scm = module_build_service.scm.SCM(source) + scm = module_build_service.scm.SCM(url) cod = scm.checkout(td) + branch = git_branch_contains(cod, commit) + git_checkout(cod, branch) + # Use configured command to create SRPM out of the SCM repo. log.debug("Creating SRPM in %s" % cod) - execute_cmd(config.mock_build_srpm_cmd.split(" "), + distgit_cmds = get_distgit_commands(source, config) + execute_cmd(distgit_cmds[1].split(" "), stdout=stdout, stderr=stderr, cwd=cod) # Find out the built SRPM and build it normally. @@ -70,6 +75,17 @@ def build_from_scm(artifact_name, source, config, build_srpm, return ret +def git_branch_contains(cod, commit): + cmd = ["git", "branch", "-r", "--contains", commit] + out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return out.strip().split("/")[1] + + +def git_checkout(cod, branch): + cmd = ["git", "checkout", branch] + execute_cmd(cmd, cwd=cod) + + def find_srpm(cod): for f in os.listdir(cod): if f.endswith(".src.rpm"): @@ -105,6 +121,13 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None): return out, err +def get_distgit_commands(source, conf): + for host, cmds in conf.distgits.items(): + if source.startswith(host): + return cmds + raise KeyError("No defined commands for {}".format(source)) + + def fake_repo_done_message(tag_name): msg = module_build_service.messaging.KojiRepoChange( msg_id='a faked internal message', From 56efd10db933e81b60d3de7b9f62399ffc186a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 30 May 2017 13:13:46 +0200 Subject: [PATCH 4/8] Allow CoprModuleBuilder to obtain packages from copr-dist-git --- module_build_service/builder/utils.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index bd0b8bbc..b5677f36 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -1,4 +1,5 @@ import os +import re import sys import koji import tempfile @@ -40,17 +41,18 @@ def build_from_scm(artifact_name, source, config, build_srpm, url, commit = source.split("?#") # Create temp dir and clone the repo there. td = tempfile.mkdtemp() - scm = module_build_service.scm.SCM(url) + scm = module_build_service.scm.SCM(source) cod = scm.checkout(td) - branch = git_branch_contains(cod, commit) - git_checkout(cod, branch) + cmd = config.mock_build_srpm_cmd.split(" ") + if is_from_copr(source): + branch = git_branch_contains(cod, commit) + git_checkout(cod, branch) + cmd = ["fedpkg-copr", "--release", branch, "srpm"] # Use configured command to create SRPM out of the SCM repo. log.debug("Creating SRPM in %s" % cod) - distgit_cmds = get_distgit_commands(source, config) - execute_cmd(distgit_cmds[1].split(" "), - stdout=stdout, stderr=stderr, cwd=cod) + execute_cmd(cmd, stdout=stdout, stderr=stderr, cwd=cod) # Find out the built SRPM and build it normally. for f in os.listdir(cod): @@ -78,7 +80,14 @@ def build_from_scm(artifact_name, source, config, build_srpm, def git_branch_contains(cod, commit): cmd = ["git", "branch", "-r", "--contains", commit] out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return out.strip().split("/")[1] + branch = out.strip().split("/")[1] + if " -> " in branch: + branch = branch.split(" -> ")[0] + return branch + + +def is_from_copr(source): + return bool(re.match("https?://copr-dist-git(-dev)?\.fedorainfracloud\.org", source)) def git_checkout(cod, branch): From 8c25f6d210dbc793ad6b363f620707b2e1a08694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 30 May 2017 14:31:35 +0200 Subject: [PATCH 5/8] Implement method for submitting a build from distgit source --- module_build_service/builder/CoprModuleBuilder.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 6a4d4b4d..02a7c29c 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -256,6 +256,15 @@ class CoprModuleBuilder(GenericBuilder): return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None + def build_scm(self, source): + # @TODO use this method once support on Copr side is finished + # Copr is currently able to create a build from fedora distgit, + # but not from custom distgit, such as copr-dist-git + url, branch = source.split("?#") + url = (url.replace("git://", "https://") + .replace("pkgs.fedoraproject.org", "src.fedoraproject.org/git")) + self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username) + def finalize(self): modulemd = tempfile.mktemp() self.module.mmd().dump(modulemd) From c7e8ba2f1db98fd43a910eebe3ef389ad68ed7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 30 May 2017 14:50:09 +0200 Subject: [PATCH 6/8] MockModuleBuilder does not use build_from_scm function anymore --- module_build_service/builder/MockModuleBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 20aa4093..6d4416b6 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -38,7 +38,7 @@ import module_build_service.scheduler import module_build_service.scheduler.consumer from base import GenericBuilder -from utils import (build_from_scm, fake_repo_done_message, +from utils import (fake_repo_done_message, create_local_repo_from_koji_tag, execute_cmd, find_srpm) from KojiModuleBuilder import KojiModuleBuilder From 0aee22b753296111664fc927c69515ab58813fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 30 May 2017 15:22:34 +0200 Subject: [PATCH 7/8] Move the hackish code to CoprModuleBuilder --- .../builder/CoprModuleBuilder.py | 80 ++++++++++++++++++- module_build_service/builder/utils.py | 80 ------------------- 2 files changed, 79 insertions(+), 81 deletions(-) diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index 02a7c29c..2efb8aa6 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -24,9 +24,12 @@ import logging import os +import re import koji import tempfile import threading +import subprocess +import shutil from copr.client import CoprClient from copr.exceptions import CoprRequestException @@ -39,7 +42,7 @@ import module_build_service.scheduler import module_build_service.scheduler.consumer from base import GenericBuilder -from utils import build_from_scm, fake_repo_done_message +from utils import execute_cmd, fake_repo_done_message from KojiModuleBuilder import KojiModuleBuilder logging.basicConfig(level=logging.DEBUG) @@ -325,3 +328,78 @@ class CoprModuleBuilder(GenericBuilder): return koji_tag.replace("+", "-") +def build_from_scm(artifact_name, source, config, build_srpm, + data=None, stdout=None, stderr=None): + """ + Builds the artifact from the SCM based source. + + :param artifact_name: Name of the artifact. + :param source: SCM URL with artifact's sources (spec file). + :param config: Config instance. + :param build_srpm: Method to call to build the RPM from the generate SRPM. + :param data: Data to be passed to the build_srpm method. + :param stdout: Python file object to which the stdout of SRPM build + command is logged. + :param stderr: Python file object to which the stderr of SRPM build + command is logged. + """ + ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM", None) + td = None + + try: + log.debug('Cloning source URL: %s' % source) + url, commit = source.split("?#") + # Create temp dir and clone the repo there. + td = tempfile.mkdtemp() + scm = module_build_service.scm.SCM(source) + cod = scm.checkout(td) + + cmd = config.mock_build_srpm_cmd.split(" ") + if is_from_copr(source): + branch = git_branch_contains(cod, commit) + git_checkout(cod, branch) + cmd = ["fedpkg-copr", "--release", branch, "srpm"] + + # Use configured command to create SRPM out of the SCM repo. + log.debug("Creating SRPM in %s" % cod) + execute_cmd(cmd, stdout=stdout, stderr=stderr, cwd=cod) + + # Find out the built SRPM and build it normally. + for f in os.listdir(cod): + if f.endswith(".src.rpm"): + log.info("Created SRPM %s" % f) + source = os.path.join(cod, f) + ret = build_srpm(artifact_name, source, data) + break + except Exception as e: + log.error("Error while generating SRPM for artifact %s: %s" % ( + artifact_name, str(e))) + ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM %s" % str(e), None) + finally: + try: + if td is not None: + shutil.rmtree(td) + except Exception as e: + log.warning( + "Failed to remove temporary directory {!r}: {}".format( + td, str(e))) + + return ret + + +def git_branch_contains(cod, commit): + cmd = ["git", "branch", "-r", "--contains", commit] + out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + branch = out.strip().split("/")[1] + if " -> " in branch: + branch = branch.split(" -> ")[0] + return branch + + +def is_from_copr(source): + return bool(re.match("https?://copr-dist-git(-dev)?\.fedorainfracloud\.org", source)) + + +def git_checkout(cod, branch): + cmd = ["git", "checkout", branch] + execute_cmd(cmd, cwd=cod) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index b5677f36..57eddcde 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -1,8 +1,5 @@ import os -import re import sys -import koji -import tempfile import shutil import subprocess import munch @@ -18,83 +15,6 @@ from module_build_service import log, scm, messaging logging.basicConfig(level=logging.DEBUG) -def build_from_scm(artifact_name, source, config, build_srpm, - data=None, stdout=None, stderr=None): - """ - Builds the artifact from the SCM based source. - - :param artifact_name: Name of the artifact. - :param source: SCM URL with artifact's sources (spec file). - :param config: Config instance. - :param build_srpm: Method to call to build the RPM from the generate SRPM. - :param data: Data to be passed to the build_srpm method. - :param stdout: Python file object to which the stdout of SRPM build - command is logged. - :param stderr: Python file object to which the stderr of SRPM build - command is logged. - """ - ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM", None) - td = None - - try: - log.debug('Cloning source URL: %s' % source) - url, commit = source.split("?#") - # Create temp dir and clone the repo there. - td = tempfile.mkdtemp() - scm = module_build_service.scm.SCM(source) - cod = scm.checkout(td) - - cmd = config.mock_build_srpm_cmd.split(" ") - if is_from_copr(source): - branch = git_branch_contains(cod, commit) - git_checkout(cod, branch) - cmd = ["fedpkg-copr", "--release", branch, "srpm"] - - # Use configured command to create SRPM out of the SCM repo. - log.debug("Creating SRPM in %s" % cod) - execute_cmd(cmd, stdout=stdout, stderr=stderr, cwd=cod) - - # Find out the built SRPM and build it normally. - for f in os.listdir(cod): - if f.endswith(".src.rpm"): - log.info("Created SRPM %s" % f) - source = os.path.join(cod, f) - ret = build_srpm(artifact_name, source, data) - break - except Exception as e: - log.error("Error while generating SRPM for artifact %s: %s" % ( - artifact_name, str(e))) - ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM %s" % str(e), None) - finally: - try: - if td is not None: - shutil.rmtree(td) - except Exception as e: - log.warning( - "Failed to remove temporary directory {!r}: {}".format( - td, str(e))) - - return ret - - -def git_branch_contains(cod, commit): - cmd = ["git", "branch", "-r", "--contains", commit] - out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - branch = out.strip().split("/")[1] - if " -> " in branch: - branch = branch.split(" -> ")[0] - return branch - - -def is_from_copr(source): - return bool(re.match("https?://copr-dist-git(-dev)?\.fedorainfracloud\.org", source)) - - -def git_checkout(cod, branch): - cmd = ["git", "checkout", branch] - execute_cmd(cmd, cwd=cod) - - def find_srpm(cod): for f in os.listdir(cod): if f.endswith(".src.rpm"): From cdb2c50266941f444fc7d431d4bb852d71197f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Thu, 1 Jun 2017 20:48:21 +0200 Subject: [PATCH 8/8] foo --- module_build_service/builder/utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index 57eddcde..f4ab6bc2 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -50,13 +50,6 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None): return out, err -def get_distgit_commands(source, conf): - for host, cmds in conf.distgits.items(): - if source.startswith(host): - return cmds - raise KeyError("No defined commands for {}".format(source)) - - def fake_repo_done_message(tag_name): msg = module_build_service.messaging.KojiRepoChange( msg_id='a faked internal message',