Mock backend: Create repository from Koji tag locally instead of using the one stored in kojipkgs.

conf/config.py

@@ -94,6 +94,8 @@ class BaseConfiguration(object):
     # Disable Client Authorization
     NO_AUTH = False
 
+    CACHE_DIR = '/var/cache/module-build-service'
+
 
 class DevConfiguration(BaseConfiguration):
     DEBUG = True

module_build_service/builder/MockModuleBuilder.py

@@ -37,7 +37,8 @@ import module_build_service.scheduler
 import module_build_service.scheduler.consumer
 
 from base import GenericBuilder
-from utils import execute_cmd, build_from_scm, fake_repo_done_message
+from utils import (build_from_scm, fake_repo_done_message,
+                   create_local_repo_from_koji_tag, execute_cmd)
 from KojiModuleBuilder import KojiModuleBuilder
 from module_build_service.models import ModuleBuild
 
@@ -274,7 +275,10 @@ mdpolicy=group:primary
         # extended to Copr in the future.
         self._load_mock_config()
         for tag in dependencies:
-            baseurl = KojiModuleBuilder.repo_from_tag(self.config, tag, self.arch)
+            repo_dir = os.path.join(self.config.cache_dir, "koji_tags", tag)
+            create_local_repo_from_koji_tag(self.config, tag, repo_dir,
+                                            [self.arch, "noarch"])
+            baseurl = "file://" + repo_dir
             self._add_repo(tag, baseurl)
         self._write_mock_config()
 

module_build_service/builder/__init__.py

@@ -9,9 +9,8 @@ __all__ = [
 
 GenericBuilder.register_backend_class(KojiModuleBuilder)
 
-if conf.system == "mock":
-    from MockModuleBuilder import MockModuleBuilder
-    GenericBuilder.register_backend_class(MockModuleBuilder)
+from MockModuleBuilder import MockModuleBuilder
+GenericBuilder.register_backend_class(MockModuleBuilder)
 
 if conf.system == "copr":
     from CoprModuleBuilder import CoprModuleBuilder

module_build_service/builder/utils.py

@@ -3,6 +3,8 @@ import koji
 import tempfile
 import shutil
 import subprocess
+import munch
+import errno
 import logging
 import module_build_service
 import module_build_service.scheduler
@@ -99,3 +101,91 @@ def fake_repo_done_message(tag_name):
         repo_tag=tag_name + "-build",
     )
     module_build_service.scheduler.consumer.work_queue_put(msg)
+
+
+def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
+    """
+    Downloads the packages build for one of `archs` (defaults to ['x86_64',
+    'noarch']) in Koji tag `tag` to `repo_dir` and creates repository in that
+    directory. Needs config.koji_profile and config.koji_config to be set.
+    """
+
+    # Placed here to avoid py2/py3 conflicts...
+    import koji
+    import urlgrabber.grabber as grabber
+    import urlgrabber.progress as progress
+
+    if not archs:
+        archs = ["x86_64", "noarch"]
+
+    # Load koji config and create Koji session.
+    koji_config = munch.Munch(koji.read_config(
+        profile_name=config.koji_profile,
+        user_config=config.koji_config,
+    ))
+
+    address = koji_config.server
+    log.info("Connecting to koji %r" % address)
+    session = koji.ClientSession(address, opts=koji_config)
+
+    # Get the list of all RPMs and builds in a tag.
+    try:
+        rpms, builds = session.listTaggedRPMS(tag, latest=True)
+    except koji.GenericError as e:
+        log.exception("Failed to list rpms in tag %r" % tag)
+
+    # Reformat builds so they are dict with build_id as a key.
+    builds = {build['build_id']: build for build in builds}
+
+    # Prepare pathinfo we will use to generate the URL.
+    pathinfo = koji.PathInfo(topdir=session.opts["topurl"])
+
+    # Prepare the list of URLs to download
+    urls = []
+    for rpm in rpms:
+        build_info = builds[rpm['build_id']]
+
+        # We do not download debuginfo packages or packages built for archs
+        # we are not interested in.
+        if koji.is_debuginfo(rpm['name']) or not rpm['arch'] in archs:
+            continue
+
+        fname = pathinfo.rpm(rpm)
+        url = pathinfo.build(build_info) + '/' + fname
+        urls.append((url, os.path.basename(fname), rpm['size']))
+
+    log.info("Downloading %d packages from Koji tag %s to %s" % (len(urls), tag, repo_dir))
+
+    # Create the output directory
+    try:
+        os.makedirs(repo_dir)
+    except OSError as exception:
+        if exception.errno != errno.EEXIST:
+            raise
+
+    # When True, we want to run the createrepo_c.
+    repo_changed = False
+
+    # Donload the RPMs.
+    pg = progress.TextMeter()
+    for url, relpath, size in urls:
+        local_fn = os.path.join(repo_dir, relpath)
+
+        # Download only when RPM is missing or the size does not match.
+        if not os.path.exists(local_fn) or os.path.getsize(local_fn) != size:
+            if os.path.exists(local_fn):
+                os.remove(local_fn)
+            repo_changed = True
+            grabber.urlgrab(url, filename=local_fn, progress_obj=pg,
+                            async=(tag, 5), text=relpath)
+
+    grabber.parallel_wait()
+
+    # If we downloaded something, run the createrepo_c.
+    if repo_changed:
+        repodata_path = os.path.join(repo_dir, "repodata")
+        if os.path.exists(repodata_path):
+            shutil.rmtree(repodata_path)
+
+        log.info("Creating local repository in %s" % repo_dir)
+        execute_cmd(['/usr/bin/createrepo_c', repo_dir])

module_build_service/config.py

@@ -127,6 +127,10 @@ class Config(object):
             'type': int,
             'default': 0,
             'desc': 'Polling interval, in seconds.'},
+        'cache_dir': {
+            'type': str,
+            'default': '/var/cache/module-build-service',
+            'desc': 'Cache directory'},
         'pdc_url': {
             'type': str,
             'default': '',