From 360a8f3b84f9cd7d85de1a796afa0d91e8c55001 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 1 Apr 2019 10:21:29 +0200 Subject: [PATCH] Add support for local component repository using `file://`. This is needed for offline local builds to build a component which is stored on local git repository. This PR also adds OfflineLocalBuildConfiguration configuration class for offline local builds to set the RESOLVER. --- conf/config.py | 6 +++ .../builder/MockModuleBuilder.py | 45 +++++++++++++++++-- module_build_service/config.py | 6 ++- module_build_service/manage.py | 4 -- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/conf/config.py b/conf/config.py index 7512dbea..12460323 100644 --- a/conf/config.py +++ b/conf/config.py @@ -139,6 +139,12 @@ class LocalBuildConfiguration(BaseConfiguration): ALLOW_CUSTOM_SCMURLS = True RESOLVER = 'mbs' + RPMS_ALLOW_REPOSITORY = True + MODULES_ALLOW_REPOSITORY = True + + +class OfflineLocalBuildConfiguration(LocalBuildConfiguration): + RESOLVER = 'local' class DevConfiguration(LocalBuildConfiguration): diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index cd1f8805..29c9c281 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -542,6 +542,25 @@ class MockModuleBuilder(GenericBuilder): if self.module.state == models.BUILD_STATES["done"]: self._createrepo(include_module_yaml=True) + @classmethod + def get_built_rpms_in_module_build(cls, mmd): + """ + :param Modulemd mmd: Modulemd to get the built RPMs from. + :return: list of NVRs + """ + with models.make_session(conf) as db_session: + build = models.ModuleBuild.get_build_from_nsvc( + db_session, mmd.get_name(), mmd.get_stream(), mmd.get_version(), + mmd.get_context()) + if build.koji_tag.startswith("repofile://"): + # Modules from local repository have already the RPMs filled in mmd. + return list(mmd.get_rpm_artifacts().get()) + else: + koji_session = KojiModuleBuilder.get_session(conf, login=False) + rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0] + nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms) + return list(nvrs) + class BaseBuilder(object): def __init__(self, config, resultsdir): @@ -565,9 +584,14 @@ class SCMBuilder(BaseBuilder): def __init__(self, config, resultsdir, source, artifact_name): super(SCMBuilder, self).__init__(config, resultsdir) with open(config, "a") as f: - branch = source.split("?#")[1] + git_repo, branch = source.split("?#") distgit_cmds = self._get_distgit_commands(source) - distgit_get = distgit_cmds[0].format(artifact_name) + + if source.startswith("file://"): + # For local git repositories, pass the full path to repository to git command. + distgit_get = distgit_cmds[0].format(git_repo) + else: + distgit_get = distgit_cmds[0].format(artifact_name) # mock-scm cannot checkout particular commit hash, but only branch. # We therefore use a command that combines the distgit-command with @@ -587,10 +611,23 @@ class SCMBuilder(BaseBuilder): artifact_name), "config_opts['scm_opts']['distgit_get'] = {!r}\n".format( distgit_get_branch), - "config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format( - distgit_cmds[1]), ]) + # Set distgit_src_get only if it's defined. + if distgit_cmds[1]: + f.write("config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format( + distgit_cmds[1])) + + # The local git repositories cloned by `fedpkg clone` typically do not have + # the tarballs with sources committed in a git repo. They normally live in lookaside + # cache on remote server, but we should not try getting them from there for true + # local builds. + # Instead, get them from local path with git repository by passing that path to Mock + # using the `ext_src_dir`. + if git_repo.startswith("file://"): + src_dir = git_repo[len("file://"):] + f.write("config_opts['scm_opts']['ext_src_dir'] = '{}'\n".format(src_dir)) + def _make_executable(self, path): mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 # copy R bits to X diff --git a/module_build_service/config.py b/module_build_service/config.py index 78b32a69..7c3dbc53 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -67,7 +67,10 @@ def init_config(app): # Load LocalBuildConfiguration section in case we are building modules # locally. if "build_module_locally" in sys.argv: - config_section = "LocalBuildConfiguration" + if "--offline"in sys.argv: + config_section = "OfflineLocalBuildConfiguration" + else: + config_section = "LocalBuildConfiguration" # try getting config_file from os.environ if 'MBS_CONFIG_FILE' in os.environ: @@ -333,6 +336,7 @@ class Config(object): 'default': { 'https://src.fedoraproject.org': ('fedpkg clone --anonymous {}', 'fedpkg --release module sources'), + 'file://': ('git clone {}', None), }, 'desc': 'Mapping between dist-git and command to '}, 'mock_config': { diff --git a/module_build_service/manage.py b/module_build_service/manage.py index 2df69a0d..b69dac50 100755 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -126,10 +126,6 @@ def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None, conf.set_item("system", "mock") conf.set_item("base_module_repofiles", platform_repofiles) - # Use the "local" resolver for offline module builds. - if offline: - conf.set_item("resolver", "local") - # Use our own local SQLite3 database. confdir = os.path.abspath(os.getcwd()) dbdir = os.path.abspath(os.path.join(confdir, '..')) if confdir.endswith('conf') \