From 6e92ed6b973e3ecd3e48f209aeac838507010214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 15 May 2017 15:34:23 +0200 Subject: [PATCH 1/8] Add function to find srpm in given directory --- module_build_service/builder/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index dbd13522..ffd77add 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -69,6 +69,12 @@ def build_from_scm(artifact_name, source, config, build_srpm, return ret +def find_srpm(cod): + for f in os.listdir(cod): + if f.endswith(".src.rpm"): + return os.path.join(cod, f) + + def execute_cmd(args, stdout=None, stderr=None, cwd=None): """ Executes command defined by `args`. If `stdout` or `stderr` is set to From 14757ec3ad1ea0448eeab5b8927132c9108919fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 15 May 2017 15:37:20 +0200 Subject: [PATCH 2/8] Implement classes for building srpms via mock --- .../builder/MockModuleBuilder.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 6c5bd4ea..1a26184a 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -429,4 +429,35 @@ mdpolicy=group:primary pass +class BaseBuilder(object): + def __init__(self, config, resultsdir): + self.config = config + self.resultsdir = resultsdir + self.cmd = ["mock", "-v", "-r", config, + "--no-clean", + "--resultdir=%s" % resultsdir] + + def build(self, stdout, stderr): + execute_cmd(self.cmd, stdout=stdout, stderr=stderr) + + +class LocalBuilder(BaseBuilder): + def __init__(self, config, resultsdir, source): + super(LocalBuilder, self).__init__(config, resultsdir) + self.cmd.extend(["--rebuild", source]) + + +class SCMBuilder(BaseBuilder): + def __init__(self, config, resultsdir, artifact_name): + super(SCMBuilder, self).__init__(config, resultsdir) + with open(config, "a") as f: + f.writelines([ + "config_opts['scm'] = True\n", + "config_opts['scm_opts']['method'] = 'distgit'\n", + "config_opts['scm_opts']['branch'] = 'f24'\n", + "config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name), + "config_opts['scm_opts']['distgit_get'] = 'fedpkg clone {} --anonymous'\n".format(artifact_name), + "config_opts['scm_opts']['distgit_src_get'] = 'fedpkg sources'\n", + ]) + From 1af400de0205fd79cfdbf0813f09495353df9852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 15 May 2017 15:41:37 +0200 Subject: [PATCH 3/8] Obtain SRPM from distgit via mock SCM --- .../builder/MockModuleBuilder.py | 48 ++++++++----------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 1a26184a..2e772cf6 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -38,7 +38,8 @@ import module_build_service.scheduler.consumer from base import GenericBuilder from utils import (build_from_scm, fake_repo_done_message, - create_local_repo_from_koji_tag, execute_cmd) + create_local_repo_from_koji_tag, execute_cmd, + find_srpm) from KojiModuleBuilder import KojiModuleBuilder from module_build_service.models import ModuleBuild @@ -305,7 +306,7 @@ mdpolicy=group:primary if os.path.exists(old_log): os.rename(old_log, new_log) - def build_srpm(self, artifact_name, source, build_id): + def build_srpm(self, artifact_name, source, build_id, builder): """ Builds the artifact from the SRPM. """ @@ -315,16 +316,6 @@ mdpolicy=group:primary mock_config = os.path.join(self.configdir, "mock-%s.cfg" % str(threading.current_thread().name)) - # Clear resultsdir associated with this thread or in case it does not - # exist, create it. - resultsdir = os.path.join(self.resultsdir, - str(threading.current_thread().name)) - if os.path.exists(resultsdir): - for name in os.listdir(resultsdir): - os.remove(os.path.join(resultsdir, name)) - else: - os.makedirs(resultsdir) - # Open the logs to which we will forward mock stdout/stderr. mock_stdout_log = open(os.path.join(self.resultsdir, artifact_name + "-mock-stdout.log"), "w") @@ -337,17 +328,16 @@ mdpolicy=group:primary stdout=mock_stdout_log, stderr=mock_stderr_log) # Start the build and store results to resultsdir - execute_cmd(["mock", "-v", "-r", mock_config, - "--no-clean", "--rebuild", source, - "--resultdir=%s" % resultsdir], - stdout=mock_stdout_log, stderr=mock_stderr_log) + resultsdir = builder.resultsdir + builder.build(mock_stdout_log, mock_stderr_log) + srpm = find_srpm(resultsdir) # Emit messages simulating complete build. These messages # are put in the scheduler's work queue and are handled # by MBS after the build_srpm() method returns and scope gets # back to scheduler.main.main() method. state = koji.BUILD_STATES['COMPLETE'] - self._send_build_change(state, source, build_id) + self._send_build_change(state, srpm, build_id) with open(os.path.join(resultsdir, "status.log"), 'w') as f: f.write("complete\n") @@ -396,6 +386,7 @@ mdpolicy=group:primary # generate the thread-specific mock config by writing it to fs again. self._load_mock_config() self._write_mock_config() + mock_config = os.path.join(self.configdir, "mock-%s.cfg" % str(threading.current_thread().name)) # Get the build-id in thread-safe manner. build_id = None @@ -403,19 +394,22 @@ mdpolicy=group:primary MockModuleBuilder._build_id += 1 build_id = int(MockModuleBuilder._build_id) + # Clear resultsdir associated with this thread or in case it does not + # exist, create it. + resultsdir = os.path.join(self.resultsdir, + str(threading.current_thread().name)) + if os.path.exists(resultsdir): + for name in os.listdir(resultsdir): + os.remove(os.path.join(resultsdir, name)) + else: + os.makedirs(resultsdir) + # Git sources are treated specially. if source.startswith(("git://", "http://", "https://")): - # Open the srpm-stdout and srpm-stderr logs and build from SCM. - srpm_stdout_fn = os.path.join(self.resultsdir, - artifact_name + "-srpm-stdout.log") - srpm_stderr_fn = os.path.join(self.resultsdir, - artifact_name + "-srpm-stderr.log") - with open(srpm_stdout_fn, "w") as srpm_stdout_log, open(srpm_stderr_fn, "w") as srpm_stderr_log: - return build_from_scm(artifact_name, source, - self.config, self.build_srpm, data=build_id, - stdout=srpm_stdout_log, stderr=srpm_stderr_log) + builder = SCMBuilder(mock_config, resultsdir, artifact_name) else: - return self.build_srpm(artifact_name, source, build_id) + builder = LocalBuilder(mock_config, resultsdir, source) + return self.build_srpm(artifact_name, source, build_id, builder) @staticmethod def get_disttag_srpm(disttag, module_build): From 8a258c3ad76e74d6f4ad6affb9c64a322176ea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 15 May 2017 17:53:03 +0200 Subject: [PATCH 4/8] Use master branch --- module_build_service/builder/MockModuleBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 2e772cf6..3f75bf83 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -448,7 +448,7 @@ class SCMBuilder(BaseBuilder): f.writelines([ "config_opts['scm'] = True\n", "config_opts['scm_opts']['method'] = 'distgit'\n", - "config_opts['scm_opts']['branch'] = 'f24'\n", + "config_opts['scm_opts']['branch'] = 'master'\n", "config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name), "config_opts['scm_opts']['distgit_get'] = 'fedpkg clone {} --anonymous'\n".format(artifact_name), "config_opts['scm_opts']['distgit_src_get'] = 'fedpkg sources'\n", From 9e4c43a3152460c54be04a02dcab5814f47e3da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 16 May 2017 15:20:28 +0200 Subject: [PATCH 5/8] Use commit hash from source as branch --- module_build_service/builder/MockModuleBuilder.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 3f75bf83..e2f07267 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -406,7 +406,7 @@ mdpolicy=group:primary # Git sources are treated specially. if source.startswith(("git://", "http://", "https://")): - builder = SCMBuilder(mock_config, resultsdir, artifact_name) + builder = SCMBuilder(mock_config, resultsdir, source, artifact_name) else: builder = LocalBuilder(mock_config, resultsdir, source) return self.build_srpm(artifact_name, source, build_id, builder) @@ -442,13 +442,14 @@ class LocalBuilder(BaseBuilder): class SCMBuilder(BaseBuilder): - def __init__(self, config, resultsdir, artifact_name): + def __init__(self, config, resultsdir, source, artifact_name): super(SCMBuilder, self).__init__(config, resultsdir) with open(config, "a") as f: + branch = source.split("?#")[1] f.writelines([ "config_opts['scm'] = True\n", "config_opts['scm_opts']['method'] = 'distgit'\n", - "config_opts['scm_opts']['branch'] = 'master'\n", + "config_opts['scm_opts']['branch'] = '{}'\n".format(branch), "config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name), "config_opts['scm_opts']['distgit_get'] = 'fedpkg clone {} --anonymous'\n".format(artifact_name), "config_opts['scm_opts']['distgit_src_get'] = 'fedpkg sources'\n", From 4c23e56181b984fb23ba216a325d561d9beb96c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 16 May 2017 16:19:56 +0200 Subject: [PATCH 6/8] Allow config values to be dict --- module_build_service/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/config.py b/module_build_service/config.py index 392c9485..7707597a 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -377,7 +377,7 @@ class Config(object): if key in self._defaults: # type conversion for configuration item convert = self._defaults[key]['type'] - if convert in [bool, int, list, str, set]: + if convert in [bool, int, list, str, set, dict]: try: # Do no try to convert None... if value is not None: From 8c8ea65b2ed7c6fe366bff91b53b3091aa772ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 16 May 2017 16:26:24 +0200 Subject: [PATCH 7/8] Don't hardcode dist-git clone command --- module_build_service/builder/MockModuleBuilder.py | 9 +++++++-- module_build_service/config.py | 6 ++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index e2f07267..b660c1a8 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -446,13 +446,18 @@ class SCMBuilder(BaseBuilder): super(SCMBuilder, self).__init__(config, resultsdir) with open(config, "a") as f: branch = source.split("?#")[1] + dist_git_get = self._get_clone_command(source).format(artifact_name) f.writelines([ "config_opts['scm'] = True\n", "config_opts['scm_opts']['method'] = 'distgit'\n", "config_opts['scm_opts']['branch'] = '{}'\n".format(branch), "config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name), - "config_opts['scm_opts']['distgit_get'] = 'fedpkg clone {} --anonymous'\n".format(artifact_name), + "config_opts['scm_opts']['distgit_get'] = '{}'\n".format(dist_git_get), "config_opts['scm_opts']['distgit_src_get'] = 'fedpkg sources'\n", ]) - + def _get_clone_command(self, source): + for host, cmd in conf.distgits.items(): + if source.startswith(host): + return cmd + raise KeyError("No defined command for {}".format(source)) diff --git a/module_build_service/config.py b/module_build_service/config.py index 7707597a..37a7dfb2 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -286,6 +286,12 @@ class Config(object): 'type': str, 'default': '', 'desc': 'Trusted certificate for ssl connection.'}, + 'distgits': { + 'type': dict, + 'default': { + 'git://pkgs.fedoraproject.org': 'fedpkg clone --anonymous {}', + }, + 'desc': 'Mapping between dist-git and command to '}, 'mock_config': { 'type': str, 'default': 'fedora-25-x86_64', From 99ba03b52d6b3e71088005405ee5816c7b90eb67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Wed, 17 May 2017 16:23:16 +0200 Subject: [PATCH 8/8] Move also distgit_src_get command to config --- module_build_service/builder/MockModuleBuilder.py | 15 ++++++++------- module_build_service/config.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index b660c1a8..af3bd68d 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -446,18 +446,19 @@ class SCMBuilder(BaseBuilder): super(SCMBuilder, self).__init__(config, resultsdir) with open(config, "a") as f: branch = source.split("?#")[1] - dist_git_get = self._get_clone_command(source).format(artifact_name) + distgit_cmds = self._get_distgit_commands(source) + distgit_get = distgit_cmds[0].format(artifact_name) f.writelines([ "config_opts['scm'] = True\n", "config_opts['scm_opts']['method'] = 'distgit'\n", "config_opts['scm_opts']['branch'] = '{}'\n".format(branch), "config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name), - "config_opts['scm_opts']['distgit_get'] = '{}'\n".format(dist_git_get), - "config_opts['scm_opts']['distgit_src_get'] = 'fedpkg sources'\n", + "config_opts['scm_opts']['distgit_get'] = '{}'\n".format(distgit_get), + "config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format(distgit_cmds[1]), ]) - def _get_clone_command(self, source): - for host, cmd in conf.distgits.items(): + def _get_distgit_commands(self, source): + for host, cmds in conf.distgits.items(): if source.startswith(host): - return cmd - raise KeyError("No defined command for {}".format(source)) + return cmds + raise KeyError("No defined commands for {}".format(source)) diff --git a/module_build_service/config.py b/module_build_service/config.py index 37a7dfb2..3a1e976a 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -289,7 +289,7 @@ class Config(object): 'distgits': { 'type': dict, 'default': { - 'git://pkgs.fedoraproject.org': 'fedpkg clone --anonymous {}', + 'git://pkgs.fedoraproject.org': ('fedpkg clone --anonymous {}', 'fedpkg sources'), }, 'desc': 'Mapping between dist-git and command to '}, 'mock_config': {