diff --git a/module_build_service/config.py b/module_build_service/config.py index e0036695..9f51d60e 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -516,6 +516,20 @@ class Config(object): 'concatenated. Any null capture groups will be ignored. The first regex that ' 'matches the branch will be used.') }, + 'default_buildroot_packages': { + 'type': list, + 'default': ["bash", "bzip2", "coreutils", "cpio", "diffutils", "findutils", "gawk", + "gcc", "gcc-c++", "grep", "gzip", "info", "make", "module-build-macros", + "patch", "fedora-release", "redhat-rpm-config", "rpm-build", "sed", + "shadow-utils", "tar", "unzip", "util-linux", "which", "xz"], + 'desc': ('The list packages for offline module build RPM buildroot.') + }, + 'default_srpm_buildroot_packages': { + 'type': list, + 'default': ["bash", "gnupg2", "module-build-macros", "fedora-release", + "redhat-rpm-config", "fedpkg-minimal", "rpm-build", "shadow-utils"], + 'desc': ('The list packages for offline module build RPM buildroot.') + }, } def __init__(self, conf_section_obj): diff --git a/module_build_service/manage.py b/module_build_service/manage.py index 0bd4584b..d3041c08 100755 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -34,7 +34,8 @@ from module_build_service import app, conf, db, create_app from module_build_service import models from module_build_service.utils import ( submit_module_build_from_yaml, - load_local_builds, load_mmd, import_mmd + load_local_builds, load_mmd, import_mmd, + import_builds_from_local_dnf_repos ) from module_build_service.errors import StreamAmbigous import module_build_service.messaging @@ -104,10 +105,12 @@ def import_module(mmd_file): @manager.option('--file', action='store', dest="yaml_file") @manager.option('--srpm', action='append', default=[], dest="srpms", metavar='SRPM') @manager.option('--skiptests', action='store_true', dest="skiptests") +@manager.option('--offline', action='store_true', dest="offline") @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs') @manager.option('-s', '--set-stream', action='append', default=[], dest='default_streams') def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None, - stream=None, skiptests=False, default_streams=None): + stream=None, skiptests=False, default_streams=None, + offline=False): """ Performs local module build using Mock """ if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']: @@ -132,6 +135,8 @@ def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None, os.remove(dbpath) db.create_all() + if offline: + import_builds_from_local_dnf_repos() load_local_builds(local_build_nsvs) params = {} diff --git a/module_build_service/utils/general.py b/module_build_service/utils/general.py index 0fd13a97..c0f2d687 100644 --- a/module_build_service/utils/general.py +++ b/module_build_service/utils/general.py @@ -29,7 +29,7 @@ import time from datetime import datetime from six import text_type -from module_build_service import conf, log, models +from module_build_service import conf, log, models, Modulemd, glib from module_build_service.errors import ( ValidationError, ProgrammingError, UnprocessableEntity) @@ -406,6 +406,98 @@ def import_mmd(session, mmd): return build, msgs +def import_fake_base_module(nsvc): + """ + Creates and imports new fake base module to be used with offline local builds. + + :param str nsvc: name:stream:version:context of a module. + """ + name, stream, version, context = nsvc.split(":") + mmd = Modulemd.Module() + mmd.set_mdversion(2) + mmd.set_name(name) + mmd.set_stream(stream) + mmd.set_version(int(version)) + mmd.set_context(context) + mmd.set_summary("fake base module") + mmd.set_description("fake base module") + licenses = Modulemd.SimpleSet() + licenses.add("GPL") + mmd.set_module_licenses(licenses) + + buildroot = Modulemd.Profile() + buildroot.set_name("buildroot") + for rpm in conf.default_buildroot_packages: + buildroot.add_rpm(rpm) + mmd.add_profile(buildroot) + + srpm_buildroot = Modulemd.Profile() + srpm_buildroot.set_name("srpm-buildroot") + for rpm in conf.default_srpm_buildroot_packages: + srpm_buildroot.add_rpm(rpm) + mmd.add_profile(srpm_buildroot) + + xmd = {'mbs': {}} + xmd_mbs = xmd['mbs'] + xmd_mbs['buildrequires'] = {} + xmd_mbs['requires'] = {} + xmd_mbs['commit'] = 'ref_%s' % context + xmd_mbs['mse'] = 'true' + xmd_mbs['koji_tag'] = 'local_build' + mmd.set_xmd(glib.dict_values(xmd)) + + with models.make_session(conf) as session: + import_mmd(session, mmd) + + +def import_builds_from_local_dnf_repos(): + """ + Imports the module builds from all available local repositories to MBS DB. + + This is used when building modules locally without any access to MBS infra. + This method also generates and imports the base module according to /etc/os-release. + """ + # Import DNF here to not force it as a hard MBS dependency. + import dnf + + log.info("Loading available RPM repositories.") + dnf_base = dnf.Base() + dnf_base.read_all_repos() + + log.info("Importing available modules to MBS local database.") + with models.make_session(conf) as session: + for repo in dnf_base.repos.values(): + try: + repo.load() + except Exception as e: + log.warning(str(e)) + continue + mmd_data = repo.get_metadata_content("modules") + mmds = Modulemd.Module.new_all_from_string(mmd_data) + for mmd in mmds: + xmd = glib.from_variant_dict(mmd.get_xmd()) + xmd["mbs"]["koji_tag"] = "local_module" + xmd["mbs"]["mse"] = True + mmd.set_xmd(glib.dict_values(xmd)) + + import_mmd(session, mmd) + + # Parse the /etc/os-release to find out the local platform:stream. + platform_id = None + with open("/etc/os-release", "r") as fd: + for l in fd.readlines(): + if not l.startswith("PLATFORM_ID"): + continue + platform_id = l.split("=")[1].strip("\"' \n") + if not platform_id: + raise ValueError("Cannot get PLATFORM_ID from /etc/os-release.") + + # Create the fake platform:stream:1:000000 module to fulfill the + # dependencies for local offline build and also to define the + # srpm-buildroot and buildroot. + import_fake_base_module("%s:1:000000" % platform_id) + + def get_mmd_from_scm(url): """ Provided an SCM URL, fetch mmd from the corresponding module YAML diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index de7651d4..bf1937fa 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -1225,3 +1225,57 @@ class TestLocalBuilds: assert len(local_modules) == 1 assert local_modules[0].koji_tag.endswith( "/module-platform-f28-3/results") + + +class TestOfflineLocalBuilds: + + def setup_method(self): + clean_database() + + def teardown_method(self): + clean_database() + + def test_import_fake_base_module(self): + module_build_service.utils.import_fake_base_module("platform:foo:1:000000") + module_build = models.ModuleBuild.get_build_from_nsvc( + db.session, "platform", "foo", 1, "000000") + assert module_build + + mmd = module_build.mmd() + xmd = glib.from_variant_dict(mmd.get_xmd()) + assert xmd == { + 'mbs': { + 'buildrequires': {}, + 'commit': 'ref_000000', + 'koji_tag': 'local_build', + 'mse': 'true', + 'requires': {}}} + + profiles = mmd.get_profiles() + assert set(profiles.keys()) == set(["buildroot", "srpm-buildroot"]) + + @patch("module_build_service.utils.general.open", create=True) + def test_import_builds_from_local_dnf_repos(self, patched_open): + pytest.importorskip("dnf") + + with patch("dnf.Base") as dnf_base: + repo = mock.MagicMock() + with open(path.join(BASE_DIR, '..', 'staged_data', 'formatted_testmodule.yaml')) as f: + repo.get_metadata_content.return_value = f.read() + base = dnf_base.return_value + base.repos = {"reponame": repo} + + patched_open.return_value = mock.mock_open( + read_data="FOO=bar\nPLATFORM_ID=platform:x\n").return_value + + module_build_service.utils.import_builds_from_local_dnf_repos() + + base.read_all_repos.assert_called_once() + repo.load.assert_called_once() + repo.get_metadata_content.assert_called_once_with("modules") + + module_build = models.ModuleBuild.get_build_from_nsvc( + db.session, "testmodule", "master", 20180205135154, "9c690d0e") + assert module_build + module_build = models.ModuleBuild.get_build_from_nsvc( + db.session, "platform", "x", 1, "000000")