From 94848511e3bf268a6b2979df4f97667a1ed30a0a Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 21 Apr 2017 10:28:33 +0200 Subject: [PATCH 1/2] Mock backend: Create repository from Koji tag locally instead of using the one stored in kojipkgs. --- conf/config.py | 2 + .../builder/MockModuleBuilder.py | 8 +- module_build_service/builder/__init__.py | 5 +- module_build_service/builder/utils.py | 90 +++++++++++++++++++ module_build_service/config.py | 4 + 5 files changed, 104 insertions(+), 5 deletions(-) diff --git a/conf/config.py b/conf/config.py index 5bf04dfd..ccaa9302 100644 --- a/conf/config.py +++ b/conf/config.py @@ -94,6 +94,8 @@ class BaseConfiguration(object): # Disable Client Authorization NO_AUTH = False + CACHE_DIR = '/var/cache/module-build-service' + class DevConfiguration(BaseConfiguration): DEBUG = True diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index a4b6a724..da680133 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -37,7 +37,8 @@ import module_build_service.scheduler import module_build_service.scheduler.consumer from base import GenericBuilder -from utils import execute_cmd, build_from_scm, fake_repo_done_message +from utils import (build_from_scm, fake_repo_done_message, + create_local_repo_from_koji_tag, execute_cmd) from KojiModuleBuilder import KojiModuleBuilder from module_build_service.models import ModuleBuild @@ -274,7 +275,10 @@ mdpolicy=group:primary # extended to Copr in the future. self._load_mock_config() for tag in dependencies: - baseurl = KojiModuleBuilder.repo_from_tag(self.config, tag, self.arch) + repo_dir = os.path.join(self.config.cache_dir, "koji_tags", tag) + create_local_repo_from_koji_tag(self.config, tag, repo_dir, + [self.arch, "noarch"]) + baseurl = "file://" + repo_dir self._add_repo(tag, baseurl) self._write_mock_config() diff --git a/module_build_service/builder/__init__.py b/module_build_service/builder/__init__.py index 99c3c8b4..be80f471 100644 --- a/module_build_service/builder/__init__.py +++ b/module_build_service/builder/__init__.py @@ -9,9 +9,8 @@ __all__ = [ GenericBuilder.register_backend_class(KojiModuleBuilder) -if conf.system == "mock": - from MockModuleBuilder import MockModuleBuilder - GenericBuilder.register_backend_class(MockModuleBuilder) +from MockModuleBuilder import MockModuleBuilder +GenericBuilder.register_backend_class(MockModuleBuilder) if conf.system == "copr": from CoprModuleBuilder import CoprModuleBuilder diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index b4126fb9..26fa4020 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -3,6 +3,8 @@ import koji import tempfile import shutil import subprocess +import munch +import errno import logging import module_build_service import module_build_service.scheduler @@ -99,3 +101,91 @@ def fake_repo_done_message(tag_name): repo_tag=tag_name + "-build", ) module_build_service.scheduler.consumer.work_queue_put(msg) + + +def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None): + """ + Downloads the packages build for one of `archs` (defaults to ['x86_64', + 'noarch']) in Koji tag `tag` to `repo_dir` and creates repository in that + directory. Needs config.koji_profile and config.koji_config to be set. + """ + + # Placed here to avoid py2/py3 conflicts... + import koji + import urlgrabber.grabber as grabber + import urlgrabber.progress as progress + + if not archs: + archs = ["x86_64", "noarch"] + + # Load koji config and create Koji session. + koji_config = munch.Munch(koji.read_config( + profile_name=config.koji_profile, + user_config=config.koji_config, + )) + + address = koji_config.server + log.info("Connecting to koji %r" % address) + session = koji.ClientSession(address, opts=koji_config) + + # Get the list of all RPMs and builds in a tag. + try: + rpms, builds = session.listTaggedRPMS(tag, latest=True) + except koji.GenericError as e: + log.exception("Failed to list rpms in tag %r" % tag) + + # Reformat builds so they are dict with build_id as a key. + builds = {build['build_id']: build for build in builds} + + # Prepare pathinfo we will use to generate the URL. + pathinfo = koji.PathInfo(topdir=session.opts["topurl"]) + + # Prepare the list of URLs to download + urls = [] + for rpm in rpms: + build_info = builds[rpm['build_id']] + + # We do not download debuginfo packages or packages built for archs + # we are not interested in. + if koji.is_debuginfo(rpm['name']) or not rpm['arch'] in archs: + continue + + fname = pathinfo.rpm(rpm) + url = pathinfo.build(build_info) + '/' + fname + urls.append((url, os.path.basename(fname), rpm['size'])) + + log.info("Downloading %d packages from Koji tag %s to %s" % (len(urls), tag, repo_dir)) + + # Create the output directory + try: + os.makedirs(repo_dir) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + + # When True, we want to run the createrepo_c. + repo_changed = False + + # Donload the RPMs. + pg = progress.TextMeter() + for url, relpath, size in urls: + local_fn = os.path.join(repo_dir, relpath) + + # Download only when RPM is missing or the size does not match. + if not os.path.exists(local_fn) or os.path.getsize(local_fn) != size: + if os.path.exists(local_fn): + os.remove(local_fn) + repo_changed = True + grabber.urlgrab(url, filename=local_fn, progress_obj=pg, + async=(tag, 5), text=relpath) + + grabber.parallel_wait() + + # If we downloaded something, run the createrepo_c. + if repo_changed: + repodata_path = os.path.join(repo_dir, "repodata") + if os.path.exists(repodata_path): + shutil.rmtree(repodata_path) + + log.info("Creating local repository in %s" % repo_dir) + execute_cmd(['/usr/bin/createrepo_c', repo_dir]) diff --git a/module_build_service/config.py b/module_build_service/config.py index 16d08624..4f58b18a 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -127,6 +127,10 @@ class Config(object): 'type': int, 'default': 0, 'desc': 'Polling interval, in seconds.'}, + 'cache_dir': { + 'type': str, + 'default': '/var/cache/module-build-service', + 'desc': 'Cache directory'}, 'pdc_url': { 'type': str, 'default': '', From ccf66804e5534aee3e3d0376cd10d30d3317f9f4 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 24 Apr 2017 15:26:52 +0200 Subject: [PATCH 2/2] Store built modules using mock in ~/modulebuild by default and use that directory for koji_tags cache by default too. --- module_build_service/builder/utils.py | 4 +-- module_build_service/config.py | 26 +++++++++++++------ tests/test_config.py | 37 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 tests/test_config.py diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index 26fa4020..65e9ee6b 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -6,6 +6,8 @@ import subprocess import munch import errno import logging +import urlgrabber.grabber as grabber +import urlgrabber.progress as progress import module_build_service import module_build_service.scheduler from module_build_service import log, scm, messaging @@ -112,8 +114,6 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None): # Placed here to avoid py2/py3 conflicts... import koji - import urlgrabber.grabber as grabber - import urlgrabber.progress as progress if not archs: archs = ["x86_64", "noarch"] diff --git a/module_build_service/config.py b/module_build_service/config.py index 4f58b18a..c4addbea 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -103,6 +103,12 @@ def init_config(app): app.config.from_object(config_section_obj) return conf +class Path: + """ + Config type for paths. Expands the users home directory. + """ + pass + class Config(object): """Class representing the orchestrator configuration.""" @@ -128,8 +134,8 @@ class Config(object): 'default': 0, 'desc': 'Polling interval, in seconds.'}, 'cache_dir': { - 'type': str, - 'default': '/var/cache/module-build-service', + 'type': Path, + 'default': '~/modulebuild/cache', 'desc': 'Cache directory'}, 'pdc_url': { 'type': str, @@ -280,8 +286,8 @@ class Config(object): 'default': 'fedpkg --release f26 srpm', 'desc': ''}, 'mock_resultsdir': { - 'type': str, - 'default': '/tmp', + 'type': Path, + 'default': '~/modulebuild/builds', 'desc': 'Directory for Mock build results.'}, 'scmurls': { 'type': list, @@ -321,7 +327,7 @@ class Config(object): # set defaults for name, values in self._defaults.items(): - self.set_item(name, values['default']) + self.set_item(name, values['default'], values['type']) # override defaults for key in dir(conf_section_obj): @@ -331,7 +337,7 @@ class Config(object): # set item (lower key) self.set_item(key.lower(), getattr(conf_section_obj, key)) - def set_item(self, key, value): + def set_item(self, key, value, value_type=None): """ Set value for configuration item. Creates the self._key = value attribute and self.key property to set/get/del the attribute. @@ -347,6 +353,10 @@ class Config(object): setifok_func = '_setifok_{}'.format(key) if hasattr(self, setifok_func): setx = lambda self, val: getattr(self, setifok_func)(val) + elif value_type == Path: + # For paths, expanduser. + setx = lambda self, val: setattr( + self, "_" + key, os.path.expanduser(val)) else: setx = lambda self, val: setattr(self, "_" + key, val) getx = lambda self: getattr(self, "_" + key) @@ -364,8 +374,8 @@ class Config(object): value = convert(value) except: raise TypeError("Configuration value conversion failed for name: %s" % key) - # unknown type/unsupported conversion - elif convert is not None: + # unknown type/unsupported conversion, or conversion not needed + elif convert is not None and convert not in [Path]: raise TypeError("Unsupported type %s for configuration item name: %s" % (convert, key)) # Set the attribute to the correct value diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..73a24ca7 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,37 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from nose.tools import raises, eq_ + + +import unittest +import mock +import os.path +from mock import patch + +from module_build_service import conf + +class TestConfig(unittest.TestCase): + def test_path_expanduser(self): + test_dir = "~/modulebuild/builds" + conf.mock_resultsdir = test_dir + self.assertEqual(conf.mock_resultsdir, os.path.expanduser(test_dir))