diff --git a/conf/config.py b/conf/config.py index 6ee310c2..cdb256d8 100644 --- a/conf/config.py +++ b/conf/config.py @@ -34,7 +34,7 @@ class BaseConfiguration(object): KOJI_TAG_PREFIXES = ['module'] KOJI_ENABLE_CONTENT_GENERATOR = True COPR_CONFIG = '/etc/module-build-service/copr.conf' - PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' + PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' PDC_INSECURE = True PDC_DEVELOP = True SCMURLS = ["git://pkgs.fedoraproject.org/modules/"] @@ -153,7 +153,7 @@ class TestConfiguration(BaseConfiguration): path.join(dbdir, 'tests', 'test_module_build_service.db')) DEBUG = True MESSAGING = 'in_memory' - PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' + PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' # Global network-related values, in seconds NET_TIMEOUT = 3 @@ -170,3 +170,7 @@ class TestConfiguration(BaseConfiguration): class ProdConfiguration(BaseConfiguration): pass + +class LocalBuildConfiguration(BaseConfiguration): + LOG_LEVEL = 'debug' + MESSAGING = 'in_memory' diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 6d4416b6..ba1c4cc7 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -164,8 +164,15 @@ mdpolicy=group:primary # Generate the mmd the same way as pungi does. m1 = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one() + m1_mmd = m1.mmd() + for rpm in os.listdir(self.resultsdir): + if not rpm.endswith(".rpm"): + continue + rpm = rpm[:-len(".rpm")] + m1_mmd.artifacts.add_rpm(str(rpm)) + mmd_path = os.path.join(path, "modules.yaml") - modulemd.dump_all(mmd_path, [ m1.mmd() ]) + modulemd.dump_all(mmd_path, [ m1_mmd ]) # Generate repo and inject modules.yaml there. execute_cmd(['/usr/bin/createrepo_c', path]) @@ -282,7 +289,10 @@ mdpolicy=group:primary self._write_mock_config() def _send_build_change(self, state, source, build_id): - nvr = kobo.rpmlib.parse_nvr(source) + try: + nvr = kobo.rpmlib.parse_nvr(source) + except ValueError: + nvr = {"name": source, "release": "unknown", "version": "unknown"} # build_id=1 and task_id=1 are OK here, because we are building just # one RPM at the time. @@ -319,13 +329,14 @@ mdpolicy=group:primary mock_stderr_log = open(os.path.join(self.resultsdir, artifact_name + "-mock-stderr.log"), "w") + srpm = artifact_name + resultsdir = builder.resultsdir try: # Initialize mock. execute_cmd(["mock", "-v", "-r", mock_config, "--init"], stdout=mock_stdout_log, stderr=mock_stderr_log) # Start the build and store results to resultsdir - resultsdir = builder.resultsdir builder.build(mock_stdout_log, mock_stderr_log) srpm = find_srpm(resultsdir) @@ -347,7 +358,7 @@ mdpolicy=group:primary # by MBS after the build_srpm() method returns and scope gets # back to scheduler.main.main() method. state = koji.BUILD_STATES['FAILED'] - self._send_build_change(state, source, + self._send_build_change(state, srpm, build_id) with open(os.path.join(resultsdir, "status.log"), 'w') as f: f.write("failed\n") @@ -445,15 +456,38 @@ class SCMBuilder(BaseBuilder): branch = source.split("?#")[1] distgit_cmds = self._get_distgit_commands(source) distgit_get = distgit_cmds[0].format(artifact_name) + + # mock-scm cannot checkout particular commit hash, but only branch. + # We therefore create distgit-clone-wrapper script which clones + # the repository and checkouts particular commit hash. + # See https://bugzilla.redhat.com/show_bug.cgi?id=1459437 for + # more info. Once mock-scm supports this feature, we can remove + # this code. + wrapper_path = os.path.join(os.path.dirname(config), "distgit-clone-wrapper") + with open(wrapper_path, "w") as fd: + fd.writelines([ + "#!/bin/sh -eu\n", + "%s\n" % distgit_get, + "git -C $1 checkout $2\n", + ]) + self._make_executable(wrapper_path) + f.writelines([ "config_opts['scm'] = True\n", "config_opts['scm_opts']['method'] = 'distgit'\n", - "config_opts['scm_opts']['branch'] = '{}'\n".format(branch), - "config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name), - "config_opts['scm_opts']['distgit_get'] = '{}'\n".format(distgit_get), - "config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format(distgit_cmds[1]), + "config_opts['scm_opts']['package'] = '{}'\n".format( + artifact_name), + "config_opts['scm_opts']['distgit_get'] = '{} {} {}'\n".format( + wrapper_path, artifact_name, branch), + "config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format( + distgit_cmds[1]), ]) + def _make_executable(self, path): + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 # copy R bits to X + os.chmod(path, mode) + def _get_distgit_commands(self, source): for host, cmds in conf.distgits.items(): if source.startswith(host): diff --git a/module_build_service/config.py b/module_build_service/config.py index f0b0eeee..ebced7fb 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -51,6 +51,12 @@ def init_config(app): if flask_app_env and any([var.startswith('mod_wsgi.') for var in app.request.environ]): config_section = 'ProdConfiguration' + + # Load LocalBuildConfiguration section in case we are building modules + # locally. + if "build_module_locally" in sys.argv: + config_section = "LocalBuildConfiguration" + # try getting config_file from os.environ if 'MBS_CONFIG_FILE' in os.environ: config_file = os.environ['MBS_CONFIG_FILE'] @@ -294,7 +300,9 @@ class Config(object): 'distgits': { 'type': dict, 'default': { - 'git://pkgs.fedoraproject.org': ('fedpkg clone --anonymous {}', 'fedpkg sources'), + 'git://pkgs.fedoraproject.org': + ('fedpkg clone --anonymous $1', + 'fedpkg --release module sources'), }, 'desc': 'Mapping between dist-git and command to '}, 'mock_config': { diff --git a/module_build_service/manage.py b/module_build_service/manage.py index 24abb4c0..4643a5d9 100644 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -152,10 +152,9 @@ def build_module_locally(url, branch): submit_module_build_from_scm(username, url, branch, allow_local_url=True) stop = module_build_service.scheduler.make_simple_stop_condition(db.session) - initial_messages = [MBSModule("local module build", 1, 1)] # Run the consumer until stop_condition returns True - module_build_service.scheduler.main(initial_messages, stop) + module_build_service.scheduler.main([], stop) @manager.command diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index ffd55ecf..b39edbd0 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -173,7 +173,7 @@ def get_module(session, module_info, strict=False): if module_info.get('active'): query['active'] = module_info['active'] - retval = session['unreleasedvariants'](page_size=-1, **query) # ordering=variant_release... + retval = session['unreleasedvariants/'](page_size=-1, **query) # ordering=variant_release... # Error handling if not retval: diff --git a/module_build_service/utils.py b/module_build_service/utils.py index d574d318..9e4a1cc7 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -107,12 +107,14 @@ def start_build_component(builder, c): Submits single component build to builder. Called in thread by QueueBasedThreadPool in continue_batch_build. """ + import koji try: c.task_id, c.state, c.state_reason, c.nvr = builder.build( artifact_name=c.package, source=c.scmurl) except Exception as e: c.state = koji.BUILD_STATES['FAILED'] c.state_reason = "Failed to build artifact %s: %s" % (c.package, str(e)) + log.exception(e) return if not c.task_id and c.state == koji.BUILD_STATES['BUILDING']: @@ -180,6 +182,11 @@ def continue_batch_build(config, module, session, builder, components=None): with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {executor.submit(start_build_component, builder, c): c for c in components_to_build} concurrent.futures.wait(futures) + # In case there has been an excepion generated directly in the + # start_build_component, the future.result() will re-raise it in the + # main thread so it is not lost. + for future in futures: + future.result() # If all components in this batch are already done, it can mean that they # have been built in the past and have been skipped in this module build. @@ -946,15 +953,15 @@ def get_reusable_component(session, module, component_name): # Perform a sanity check to make sure that the buildrequires are the same # as the buildrequires in xmd for the passed in mmd - if mmd.buildrequires.keys() != mmd.xmd['mbs']['buildrequires'].keys(): + if set(mmd.buildrequires.keys()) != set(mmd.xmd['mbs']['buildrequires'].keys()): log.error( 'The submitted module "{0}" has different keys in mmd.buildrequires' ' than in mmd.xmd[\'mbs\'][\'buildrequires\']'.format(mmd.name)) return None # Perform a sanity check to make sure that the buildrequires are the same # as the buildrequires in xmd for the mmd of the previous module build - if old_mmd.buildrequires.keys() != \ - old_mmd.xmd['mbs']['buildrequires'].keys(): + if set(old_mmd.buildrequires.keys()) != \ + set(old_mmd.xmd['mbs']['buildrequires'].keys()): log.error( 'Version "{0}" of the module "{1}" has different keys in ' 'mmd.buildrequires than in mmd.xmd[\'mbs\'][\'buildrequires\']'