Merge #518 Mock backend: Create repository from Koji tag locally instead of using the one stored in kojipkgs.

conf/config.py

@@ -94,6 +94,8 @@ class BaseConfiguration(object):
     # Disable Client Authorization
     NO_AUTH = False
 
+    CACHE_DIR = '/var/cache/module-build-service'
+
 
 class DevConfiguration(BaseConfiguration):
     DEBUG = True

module_build_service/builder/MockModuleBuilder.py

@@ -37,7 +37,8 @@ import module_build_service.scheduler
 import module_build_service.scheduler.consumer
 
 from base import GenericBuilder
-from utils import execute_cmd, build_from_scm, fake_repo_done_message
+from utils import (build_from_scm, fake_repo_done_message,
+                   create_local_repo_from_koji_tag, execute_cmd)
 from KojiModuleBuilder import KojiModuleBuilder
 from module_build_service.models import ModuleBuild
 
@@ -274,7 +275,10 @@ mdpolicy=group:primary
         # extended to Copr in the future.
         self._load_mock_config()
         for tag in dependencies:
-            baseurl = KojiModuleBuilder.repo_from_tag(self.config, tag, self.arch)
+            repo_dir = os.path.join(self.config.cache_dir, "koji_tags", tag)
+            create_local_repo_from_koji_tag(self.config, tag, repo_dir,
+                                            [self.arch, "noarch"])
+            baseurl = "file://" + repo_dir
             self._add_repo(tag, baseurl)
         self._write_mock_config()
 

module_build_service/builder/__init__.py

@@ -9,9 +9,8 @@ __all__ = [
 
 GenericBuilder.register_backend_class(KojiModuleBuilder)
 
-if conf.system == "mock":
-    from MockModuleBuilder import MockModuleBuilder
-    GenericBuilder.register_backend_class(MockModuleBuilder)
+from MockModuleBuilder import MockModuleBuilder
+GenericBuilder.register_backend_class(MockModuleBuilder)
 
 if conf.system == "copr":
     from CoprModuleBuilder import CoprModuleBuilder

module_build_service/builder/utils.py

@@ -3,7 +3,11 @@ import koji
 import tempfile
 import shutil
 import subprocess
+import munch
+import errno
 import logging
+import urlgrabber.grabber as grabber
+import urlgrabber.progress as progress
 import module_build_service
 import module_build_service.scheduler
 from module_build_service import log, scm, messaging
@@ -99,3 +103,89 @@ def fake_repo_done_message(tag_name):
         repo_tag=tag_name + "-build",
     )
     module_build_service.scheduler.consumer.work_queue_put(msg)
+
+
+def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
+    """
+    Downloads the packages build for one of `archs` (defaults to ['x86_64',
+    'noarch']) in Koji tag `tag` to `repo_dir` and creates repository in that
+    directory. Needs config.koji_profile and config.koji_config to be set.
+    """
+
+    # Placed here to avoid py2/py3 conflicts...
+    import koji
+
+    if not archs:
+        archs = ["x86_64", "noarch"]
+
+    # Load koji config and create Koji session.
+    koji_config = munch.Munch(koji.read_config(
+        profile_name=config.koji_profile,
+        user_config=config.koji_config,
+    ))
+
+    address = koji_config.server
+    log.info("Connecting to koji %r" % address)
+    session = koji.ClientSession(address, opts=koji_config)
+
+    # Get the list of all RPMs and builds in a tag.
+    try:
+        rpms, builds = session.listTaggedRPMS(tag, latest=True)
+    except koji.GenericError as e:
+        log.exception("Failed to list rpms in tag %r" % tag)
+
+    # Reformat builds so they are dict with build_id as a key.
+    builds = {build['build_id']: build for build in builds}
+
+    # Prepare pathinfo we will use to generate the URL.
+    pathinfo = koji.PathInfo(topdir=session.opts["topurl"])
+
+    # Prepare the list of URLs to download
+    urls = []
+    for rpm in rpms:
+        build_info = builds[rpm['build_id']]
+
+        # We do not download debuginfo packages or packages built for archs
+        # we are not interested in.
+        if koji.is_debuginfo(rpm['name']) or not rpm['arch'] in archs:
+            continue
+
+        fname = pathinfo.rpm(rpm)
+        url = pathinfo.build(build_info) + '/' + fname
+        urls.append((url, os.path.basename(fname), rpm['size']))
+
+    log.info("Downloading %d packages from Koji tag %s to %s" % (len(urls), tag, repo_dir))
+
+    # Create the output directory
+    try:
+        os.makedirs(repo_dir)
+    except OSError as exception:
+        if exception.errno != errno.EEXIST:
+            raise
+
+    # When True, we want to run the createrepo_c.
+    repo_changed = False
+
+    # Donload the RPMs.
+    pg = progress.TextMeter()
+    for url, relpath, size in urls:
+        local_fn = os.path.join(repo_dir, relpath)
+
+        # Download only when RPM is missing or the size does not match.
+        if not os.path.exists(local_fn) or os.path.getsize(local_fn) != size:
+            if os.path.exists(local_fn):
+                os.remove(local_fn)
+            repo_changed = True
+            grabber.urlgrab(url, filename=local_fn, progress_obj=pg,
+                            async=(tag, 5), text=relpath)
+
+    grabber.parallel_wait()
+
+    # If we downloaded something, run the createrepo_c.
+    if repo_changed:
+        repodata_path = os.path.join(repo_dir, "repodata")
+        if os.path.exists(repodata_path):
+            shutil.rmtree(repodata_path)
+
+        log.info("Creating local repository in %s" % repo_dir)
+        execute_cmd(['/usr/bin/createrepo_c', repo_dir])

module_build_service/config.py

@@ -103,6 +103,12 @@ def init_config(app):
     app.config.from_object(config_section_obj)
     return conf
 
+class Path:
+    """
+    Config type for paths. Expands the users home directory.
+    """
+    pass
+
 
 class Config(object):
     """Class representing the orchestrator configuration."""
@@ -127,6 +133,10 @@ class Config(object):
             'type': int,
             'default': 0,
             'desc': 'Polling interval, in seconds.'},
+        'cache_dir': {
+            'type': Path,
+            'default': '~/modulebuild/cache',
+            'desc': 'Cache directory'},
         'pdc_url': {
             'type': str,
             'default': '',
@@ -276,8 +286,8 @@ class Config(object):
             'default': 'fedpkg --release f26 srpm',
             'desc': ''},
         'mock_resultsdir': {
-            'type': str,
-            'default': '/tmp',
+            'type': Path,
+            'default': '~/modulebuild/builds',
             'desc': 'Directory for Mock build results.'},
         'scmurls': {
             'type': list,
@@ -317,7 +327,7 @@ class Config(object):
 
         # set defaults
         for name, values in self._defaults.items():
-            self.set_item(name, values['default'])
+            self.set_item(name, values['default'], values['type'])
 
         # override defaults
         for key in dir(conf_section_obj):
@@ -327,7 +337,7 @@ class Config(object):
             # set item (lower key)
             self.set_item(key.lower(), getattr(conf_section_obj, key))
 
-    def set_item(self, key, value):
+    def set_item(self, key, value, value_type=None):
         """
         Set value for configuration item. Creates the self._key = value
         attribute and self.key property to set/get/del the attribute.
@@ -343,6 +353,10 @@ class Config(object):
         setifok_func = '_setifok_{}'.format(key)
         if hasattr(self, setifok_func):
             setx = lambda self, val: getattr(self, setifok_func)(val)
+        elif value_type == Path:
+            # For paths, expanduser.
+            setx = lambda self, val: setattr(
+                self, "_" + key, os.path.expanduser(val))
         else:
             setx = lambda self, val: setattr(self, "_" + key, val)
         getx = lambda self: getattr(self, "_" + key)
@@ -360,8 +374,8 @@ class Config(object):
                         value = convert(value)
                 except:
                     raise TypeError("Configuration value conversion failed for name: %s" % key)
-            # unknown type/unsupported conversion
-            elif convert is not None:
+            # unknown type/unsupported conversion, or conversion not needed
+            elif convert is not None and convert not in [Path]:
                 raise TypeError("Unsupported type %s for configuration item name: %s" % (convert, key))
 
         # Set the attribute to the correct value

tests/test_config.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2016  Red Hat, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Written by Jan Kaluza <jkaluza@redhat.com>
+
+from nose.tools import raises, eq_
+
+
+import unittest
+import mock
+import os.path
+from mock import patch
+
+from module_build_service import conf
+
+class TestConfig(unittest.TestCase):
+    def test_path_expanduser(self):
+        test_dir = "~/modulebuild/builds"
+        conf.mock_resultsdir = test_dir
+        self.assertEqual(conf.mock_resultsdir, os.path.expanduser(test_dir))