From 94848511e3bf268a6b2979df4f97667a1ed30a0a Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 21 Apr 2017 10:28:33 +0200 Subject: [PATCH] Mock backend: Create repository from Koji tag locally instead of using the one stored in kojipkgs. --- conf/config.py | 2 + .../builder/MockModuleBuilder.py | 8 +- module_build_service/builder/__init__.py | 5 +- module_build_service/builder/utils.py | 90 +++++++++++++++++++ module_build_service/config.py | 4 + 5 files changed, 104 insertions(+), 5 deletions(-) diff --git a/conf/config.py b/conf/config.py index 5bf04dfd..ccaa9302 100644 --- a/conf/config.py +++ b/conf/config.py @@ -94,6 +94,8 @@ class BaseConfiguration(object): # Disable Client Authorization NO_AUTH = False + CACHE_DIR = '/var/cache/module-build-service' + class DevConfiguration(BaseConfiguration): DEBUG = True diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index a4b6a724..da680133 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -37,7 +37,8 @@ import module_build_service.scheduler import module_build_service.scheduler.consumer from base import GenericBuilder -from utils import execute_cmd, build_from_scm, fake_repo_done_message +from utils import (build_from_scm, fake_repo_done_message, + create_local_repo_from_koji_tag, execute_cmd) from KojiModuleBuilder import KojiModuleBuilder from module_build_service.models import ModuleBuild @@ -274,7 +275,10 @@ mdpolicy=group:primary # extended to Copr in the future. self._load_mock_config() for tag in dependencies: - baseurl = KojiModuleBuilder.repo_from_tag(self.config, tag, self.arch) + repo_dir = os.path.join(self.config.cache_dir, "koji_tags", tag) + create_local_repo_from_koji_tag(self.config, tag, repo_dir, + [self.arch, "noarch"]) + baseurl = "file://" + repo_dir self._add_repo(tag, baseurl) self._write_mock_config() diff --git a/module_build_service/builder/__init__.py b/module_build_service/builder/__init__.py index 99c3c8b4..be80f471 100644 --- a/module_build_service/builder/__init__.py +++ b/module_build_service/builder/__init__.py @@ -9,9 +9,8 @@ __all__ = [ GenericBuilder.register_backend_class(KojiModuleBuilder) -if conf.system == "mock": - from MockModuleBuilder import MockModuleBuilder - GenericBuilder.register_backend_class(MockModuleBuilder) +from MockModuleBuilder import MockModuleBuilder +GenericBuilder.register_backend_class(MockModuleBuilder) if conf.system == "copr": from CoprModuleBuilder import CoprModuleBuilder diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index b4126fb9..26fa4020 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -3,6 +3,8 @@ import koji import tempfile import shutil import subprocess +import munch +import errno import logging import module_build_service import module_build_service.scheduler @@ -99,3 +101,91 @@ def fake_repo_done_message(tag_name): repo_tag=tag_name + "-build", ) module_build_service.scheduler.consumer.work_queue_put(msg) + + +def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None): + """ + Downloads the packages build for one of `archs` (defaults to ['x86_64', + 'noarch']) in Koji tag `tag` to `repo_dir` and creates repository in that + directory. Needs config.koji_profile and config.koji_config to be set. + """ + + # Placed here to avoid py2/py3 conflicts... + import koji + import urlgrabber.grabber as grabber + import urlgrabber.progress as progress + + if not archs: + archs = ["x86_64", "noarch"] + + # Load koji config and create Koji session. + koji_config = munch.Munch(koji.read_config( + profile_name=config.koji_profile, + user_config=config.koji_config, + )) + + address = koji_config.server + log.info("Connecting to koji %r" % address) + session = koji.ClientSession(address, opts=koji_config) + + # Get the list of all RPMs and builds in a tag. + try: + rpms, builds = session.listTaggedRPMS(tag, latest=True) + except koji.GenericError as e: + log.exception("Failed to list rpms in tag %r" % tag) + + # Reformat builds so they are dict with build_id as a key. + builds = {build['build_id']: build for build in builds} + + # Prepare pathinfo we will use to generate the URL. + pathinfo = koji.PathInfo(topdir=session.opts["topurl"]) + + # Prepare the list of URLs to download + urls = [] + for rpm in rpms: + build_info = builds[rpm['build_id']] + + # We do not download debuginfo packages or packages built for archs + # we are not interested in. + if koji.is_debuginfo(rpm['name']) or not rpm['arch'] in archs: + continue + + fname = pathinfo.rpm(rpm) + url = pathinfo.build(build_info) + '/' + fname + urls.append((url, os.path.basename(fname), rpm['size'])) + + log.info("Downloading %d packages from Koji tag %s to %s" % (len(urls), tag, repo_dir)) + + # Create the output directory + try: + os.makedirs(repo_dir) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + + # When True, we want to run the createrepo_c. + repo_changed = False + + # Donload the RPMs. + pg = progress.TextMeter() + for url, relpath, size in urls: + local_fn = os.path.join(repo_dir, relpath) + + # Download only when RPM is missing or the size does not match. + if not os.path.exists(local_fn) or os.path.getsize(local_fn) != size: + if os.path.exists(local_fn): + os.remove(local_fn) + repo_changed = True + grabber.urlgrab(url, filename=local_fn, progress_obj=pg, + async=(tag, 5), text=relpath) + + grabber.parallel_wait() + + # If we downloaded something, run the createrepo_c. + if repo_changed: + repodata_path = os.path.join(repo_dir, "repodata") + if os.path.exists(repodata_path): + shutil.rmtree(repodata_path) + + log.info("Creating local repository in %s" % repo_dir) + execute_cmd(['/usr/bin/createrepo_c', repo_dir]) diff --git a/module_build_service/config.py b/module_build_service/config.py index 16d08624..4f58b18a 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -127,6 +127,10 @@ class Config(object): 'type': int, 'default': 0, 'desc': 'Polling interval, in seconds.'}, + 'cache_dir': { + 'type': str, + 'default': '/var/cache/module-build-service', + 'desc': 'Cache directory'}, 'pdc_url': { 'type': str, 'default': '',