Import modules found in the local repositories to MBS database for Offline local builds.

This is the first PR in many for Offline local builds. This PR:

- Adds --offline flag to build_module_locally mbs-manager command to enable
  offline local builds.
- If this flag is used, new `import_builds_from_local_dnf_repos` method is
  called which uses DNF API to get all the available installable modulemd
  files and imports each module into MBS local SQLite database.
- It also adds fake "platform:stream" module based on the /etc/os-release,
  so the buildrequirements of the imported modules are satisfied.

The idea here is that in upcoming commits, I will create LocalResolver
which will be similar to DBResolver with some extra rules to resolve
local module builds. This new LocalResolver will still be based on
the models.ModuleBuild methods and therefore we need the modules
imported in database.

module_build_service/config.py

@@ -516,6 +516,20 @@ class Config(object):
                      'concatenated. Any null capture groups will be ignored. The first regex that '
                      'matches the branch will be used.')
         },
+        'default_buildroot_packages': {
+            'type': list,
+            'default': ["bash", "bzip2", "coreutils", "cpio", "diffutils", "findutils", "gawk",
+                        "gcc", "gcc-c++", "grep", "gzip", "info", "make", "module-build-macros",
+                        "patch", "fedora-release", "redhat-rpm-config", "rpm-build", "sed",
+                        "shadow-utils", "tar", "unzip", "util-linux", "which", "xz"],
+            'desc': ('The list packages for offline module build RPM buildroot.')
+        },
+        'default_srpm_buildroot_packages': {
+            'type': list,
+            'default': ["bash", "gnupg2", "module-build-macros", "fedora-release",
+                        "redhat-rpm-config", "fedpkg-minimal", "rpm-build", "shadow-utils"],
+            'desc': ('The list packages for offline module build RPM buildroot.')
+        },
     }
 
     def __init__(self, conf_section_obj):

module_build_service/manage.py

@@ -34,7 +34,8 @@ from module_build_service import app, conf, db, create_app
 from module_build_service import models
 from module_build_service.utils import (
     submit_module_build_from_yaml,
-    load_local_builds, load_mmd, import_mmd
+    load_local_builds, load_mmd, import_mmd,
+    import_builds_from_local_dnf_repos
 )
 from module_build_service.errors import StreamAmbigous
 import module_build_service.messaging
@@ -104,10 +105,12 @@ def import_module(mmd_file):
 @manager.option('--file', action='store', dest="yaml_file")
 @manager.option('--srpm', action='append', default=[], dest="srpms", metavar='SRPM')
 @manager.option('--skiptests', action='store_true', dest="skiptests")
+@manager.option('--offline', action='store_true', dest="offline")
 @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs')
 @manager.option('-s', '--set-stream', action='append', default=[], dest='default_streams')
 def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None,
-                         stream=None, skiptests=False, default_streams=None):
+                         stream=None, skiptests=False, default_streams=None,
+                         offline=False):
     """ Performs local module build using Mock
     """
     if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']:
@@ -132,6 +135,8 @@ def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None,
             os.remove(dbpath)
 
         db.create_all()
+        if offline:
+            import_builds_from_local_dnf_repos()
         load_local_builds(local_build_nsvs)
 
         params = {}

module_build_service/utils/general.py

@@ -29,7 +29,7 @@ import time
 from datetime import datetime
 from six import text_type
 
-from module_build_service import conf, log, models
+from module_build_service import conf, log, models, Modulemd, glib
 from module_build_service.errors import (
     ValidationError, ProgrammingError, UnprocessableEntity)
 
@@ -406,6 +406,98 @@ def import_mmd(session, mmd):
     return build, msgs
 
 
+def import_fake_base_module(nsvc):
+    """
+    Creates and imports new fake base module to be used with offline local builds.
+
+    :param str nsvc: name:stream:version:context of a module.
+    """
+    name, stream, version, context = nsvc.split(":")
+    mmd = Modulemd.Module()
+    mmd.set_mdversion(2)
+    mmd.set_name(name)
+    mmd.set_stream(stream)
+    mmd.set_version(int(version))
+    mmd.set_context(context)
+    mmd.set_summary("fake base module")
+    mmd.set_description("fake base module")
+    licenses = Modulemd.SimpleSet()
+    licenses.add("GPL")
+    mmd.set_module_licenses(licenses)
+
+    buildroot = Modulemd.Profile()
+    buildroot.set_name("buildroot")
+    for rpm in conf.default_buildroot_packages:
+        buildroot.add_rpm(rpm)
+    mmd.add_profile(buildroot)
+
+    srpm_buildroot = Modulemd.Profile()
+    srpm_buildroot.set_name("srpm-buildroot")
+    for rpm in conf.default_srpm_buildroot_packages:
+        srpm_buildroot.add_rpm(rpm)
+    mmd.add_profile(srpm_buildroot)
+
+    xmd = {'mbs': {}}
+    xmd_mbs = xmd['mbs']
+    xmd_mbs['buildrequires'] = {}
+    xmd_mbs['requires'] = {}
+    xmd_mbs['commit'] = 'ref_%s' % context
+    xmd_mbs['mse'] = 'true'
+    xmd_mbs['koji_tag'] = 'local_build'
+    mmd.set_xmd(glib.dict_values(xmd))
+
+    with models.make_session(conf) as session:
+        import_mmd(session, mmd)
+
+
+def import_builds_from_local_dnf_repos():
+    """
+    Imports the module builds from all available local repositories to MBS DB.
+
+    This is used when building modules locally without any access to MBS infra.
+    This method also generates and imports the base module according to /etc/os-release.
+    """
+    # Import DNF here to not force it as a hard MBS dependency.
+    import dnf
+
+    log.info("Loading available RPM repositories.")
+    dnf_base = dnf.Base()
+    dnf_base.read_all_repos()
+
+    log.info("Importing available modules to MBS local database.")
+    with models.make_session(conf) as session:
+        for repo in dnf_base.repos.values():
+            try:
+                repo.load()
+            except Exception as e:
+                log.warning(str(e))
+                continue
+            mmd_data = repo.get_metadata_content("modules")
+            mmds = Modulemd.Module.new_all_from_string(mmd_data)
+            for mmd in mmds:
+                xmd = glib.from_variant_dict(mmd.get_xmd())
+                xmd["mbs"]["koji_tag"] = "local_module"
+                xmd["mbs"]["mse"] = True
+                mmd.set_xmd(glib.dict_values(xmd))
+
+                import_mmd(session, mmd)
+
+    # Parse the /etc/os-release to find out the local platform:stream.
+    platform_id = None
+    with open("/etc/os-release", "r") as fd:
+        for l in fd.readlines():
+            if not l.startswith("PLATFORM_ID"):
+                continue
+            platform_id = l.split("=")[1].strip("\"' \n")
+    if not platform_id:
+        raise ValueError("Cannot get PLATFORM_ID from /etc/os-release.")
+
+    # Create the fake platform:stream:1:000000 module to fulfill the
+    # dependencies for local offline build and also to define the
+    # srpm-buildroot and buildroot.
+    import_fake_base_module("%s:1:000000" % platform_id)
+
+
 def get_mmd_from_scm(url):
     """
     Provided an SCM URL, fetch mmd from the corresponding module YAML

tests/test_utils/test_utils.py

@@ -1225,3 +1225,57 @@ class TestLocalBuilds:
             assert len(local_modules) == 1
             assert local_modules[0].koji_tag.endswith(
                 "/module-platform-f28-3/results")
+
+
+class TestOfflineLocalBuilds:
+
+    def setup_method(self):
+        clean_database()
+
+    def teardown_method(self):
+        clean_database()
+
+    def test_import_fake_base_module(self):
+        module_build_service.utils.import_fake_base_module("platform:foo:1:000000")
+        module_build = models.ModuleBuild.get_build_from_nsvc(
+            db.session, "platform", "foo", 1, "000000")
+        assert module_build
+
+        mmd = module_build.mmd()
+        xmd = glib.from_variant_dict(mmd.get_xmd())
+        assert xmd == {
+            'mbs': {
+                'buildrequires': {},
+                'commit': 'ref_000000',
+                'koji_tag': 'local_build',
+                'mse': 'true',
+                'requires': {}}}
+
+        profiles = mmd.get_profiles()
+        assert set(profiles.keys()) == set(["buildroot", "srpm-buildroot"])
+
+    @patch("module_build_service.utils.general.open", create=True)
+    def test_import_builds_from_local_dnf_repos(self, patched_open):
+        pytest.importorskip("dnf")
+
+        with patch("dnf.Base") as dnf_base:
+            repo = mock.MagicMock()
+            with open(path.join(BASE_DIR, '..', 'staged_data', 'formatted_testmodule.yaml')) as f:
+                repo.get_metadata_content.return_value = f.read()
+            base = dnf_base.return_value
+            base.repos = {"reponame": repo}
+
+            patched_open.return_value = mock.mock_open(
+                read_data="FOO=bar\nPLATFORM_ID=platform:x\n").return_value
+
+            module_build_service.utils.import_builds_from_local_dnf_repos()
+
+            base.read_all_repos.assert_called_once()
+            repo.load.assert_called_once()
+            repo.get_metadata_content.assert_called_once_with("modules")
+
+            module_build = models.ModuleBuild.get_build_from_nsvc(
+                db.session, "testmodule", "master", 20180205135154, "9c690d0e")
+            assert module_build
+            module_build = models.ModuleBuild.get_build_from_nsvc(
+                db.session, "platform", "x", 1, "000000")