Allow building module in --offline module with dependencies from local repositories.

There are following changes introduced in this commit:

- The `koji_tag` of module builds imported from the local repositories
  is now in `repofile:///etc/yum.repos.d/some.repo` format to store the
  repository from which the module was imported to local MBS DB.
- The `koji_tag` of fake base module is set to empty `repofile://`
  and in `MockModuleBuilder` the `conf.base_module_repofiles` list
  is used as source for the repositories defining platform. We can't
  simply use single repository, because there might be fedora.repo
  and fedora-update.repo and so on.
- The list of default .repo files for platform are passed using the
  `-r` switch in `build_module_locally` `mbs-manager` command.
- The LocalResolver (subclass of DBResolver) is added which is used
  to resolve the build dependencies when building modules offline
  locally.
- The `MockModuleBuilder` enables the buildrequired modules and
  repositories from which they come in the mock config.

With this commit, it is possible to build testmodule locally
without any external infra.
This commit is contained in:
Jan Kaluza
2019-03-29 11:31:20 +01:00
committed by mprahl
parent e43d979bc8
commit 67a5a9d1b0
15 changed files with 263 additions and 32 deletions

View File

@@ -8,6 +8,7 @@ config_opts['releasever'] = ''
config_opts['package_manager'] = 'dnf'
config_opts['nosync'] = True
config_opts['use_bootstrap_container'] = False
config_opts['module_enable'] = $enabled_modules
config_opts['yum.conf'] = """
$yum_conf

View File

@@ -534,8 +534,9 @@ chmod 644 %buildroot/etc/rpm/macros.zz-modules
log.info("%r buildroot sucessfully connected." % self)
def buildroot_add_repos(self, dependencies):
log.info("%r adding deps on %r" % (self, dependencies))
self._koji_add_many_tag_inheritance(self.module_build_tag, dependencies)
koji_tags = dependencies.keys()
log.info("%r adding deps on %r" % (self, koji_tags))
self._koji_add_many_tag_inheritance(self.module_build_tag, koji_tags)
def _get_tagged_nvrs(self, tag):
"""

View File

@@ -90,6 +90,7 @@ class MockModuleBuilder(GenericBuilder):
self.tag_name = tag_name
self.config = config
self.groups = []
self.enabled_modules = []
self.yum_conf = MockModuleBuilder.yum_config_template
self.koji_session = None
@@ -170,8 +171,7 @@ class MockModuleBuilder(GenericBuilder):
pkglist_f = open(pkglist, "w")
# Generate the mmd the same way as pungi does.
m1 = models.ModuleBuild.query.filter(models.ModuleBuild.name == self.module_str).one()
m1_mmd = m1.mmd()
m1_mmd = self.module.mmd()
artifacts = Modulemd.SimpleSet()
rpm_files = [f
@@ -192,7 +192,7 @@ class MockModuleBuilder(GenericBuilder):
for rpm_file, nevra in zip(rpm_files, nevras):
name, epoch, version, release, arch = nevra.split()
if m1.last_batch_id() == m1.batch:
if self.module.last_batch_id() == self.module.batch:
# If RPM is filtered-out, do not add it to artifacts list.
if name in m1_mmd.get_rpm_filter().get():
continue
@@ -223,6 +223,14 @@ class MockModuleBuilder(GenericBuilder):
self.yum_conf += extra
self.yum_conf += "enabled=1\n\n"
def _add_repo_from_path(self, path):
"""
Adds repository stored in `path` to Mock config file. Call _write_mock_config() to
actually write the config file to filesystem.
"""
with open(path) as fd:
self.yum_conf += fd.read()
def _load_mock_config(self):
"""
Loads the variables which are generated only during the first
@@ -248,6 +256,7 @@ class MockModuleBuilder(GenericBuilder):
self.groups = config_opts["chroot_setup_cmd"].split(" ")[1:]
self.yum_conf = config_opts['yum.conf']
self.enabled_modules = config_opts['module_enable']
def _write_mock_config(self):
"""
@@ -261,6 +270,7 @@ class MockModuleBuilder(GenericBuilder):
config = config.replace("$arch", self.arch)
config = config.replace("$group", " ".join(self.groups))
config = config.replace("$yum_conf", self.yum_conf)
config = config.replace("$enabled_modules", str(self.enabled_modules))
# We write the most recent config to "mock.cfg", so thread-related
# configs can be later (re-)generated from it using _load_mock_config.
@@ -313,17 +323,32 @@ class MockModuleBuilder(GenericBuilder):
def buildroot_add_repos(self, dependencies):
self._load_mock_config()
for tag in dependencies:
# If tag starts with mock_resultdir, it means it is path to local
for source, mmds in dependencies.items():
# If source starts with mock_resultdir, it means it is path to local
# module build repository.
if tag.startswith(conf.mock_resultsdir):
repo_name = os.path.basename(tag)
if source.startswith(conf.mock_resultsdir):
repo_name = os.path.basename(source)
if repo_name.startswith("module-"):
repo_name = repo_name[7:]
repo_dir = tag
repo_dir = source
baseurl = "file://" + repo_dir
# If source starts with "repofile://", it is path to local /etc/yum.repos.d
# repo file.
elif source.startswith("repofile://"):
# For the base module, we want to include all the `conf.base_module_repofiles`.
if len(mmds) == 1 and mmds[0].get_name() in conf.base_module_names:
for repofile in conf.base_module_repofiles:
self._add_repo_from_path(repofile)
else:
# Add repositories defined in repofile to mock config.
repofile = source[len("repofile://"):]
self._add_repo_from_path(repofile)
# Enabled all the modular dependencies by default in Mock.
for mmd in mmds:
self.enabled_modules.append("%s:%s" % (mmd.get_name(), mmd.get_stream()))
continue
else:
repo_name = tag
repo_name = tag = source
koji_config = get_koji_config(self.config)
koji_session = koji.ClientSession(koji_config.server, opts=koji_config)
repo = koji_session.getRepo(repo_name)

View File

@@ -39,7 +39,8 @@ SUPPORTED_STRATEGIES = ['changed-and-after', 'only-changed', 'all']
SUPPORTED_RESOLVERS = {
'mbs': {'builders': ['mock']},
'db': {'builders': ['koji', 'mock', 'copr']}
'db': {'builders': ['koji', 'mock', 'copr']},
'local': {'builders': ['mock']},
}
@@ -519,14 +520,14 @@ class Config(object):
'default_buildroot_packages': {
'type': list,
'default': ["bash", "bzip2", "coreutils", "cpio", "diffutils", "findutils", "gawk",
"gcc", "gcc-c++", "grep", "gzip", "info", "make", "module-build-macros",
"gcc", "gcc-c++", "grep", "gzip", "info", "make",
"patch", "fedora-release", "redhat-rpm-config", "rpm-build", "sed",
"shadow-utils", "tar", "unzip", "util-linux", "which", "xz"],
'desc': ('The list packages for offline module build RPM buildroot.')
},
'default_srpm_buildroot_packages': {
'type': list,
'default': ["bash", "gnupg2", "module-build-macros", "fedora-release",
'default': ["bash", "gnupg2", "fedora-release",
"redhat-rpm-config", "fedpkg-minimal", "rpm-build", "shadow-utils"],
'desc': ('The list packages for offline module build RPM buildroot.')
},

View File

@@ -108,9 +108,11 @@ def import_module(mmd_file):
@manager.option('--offline', action='store_true', dest="offline")
@manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs')
@manager.option('-s', '--set-stream', action='append', default=[], dest='default_streams')
@manager.option('-r', '--platform-repo-file', action='append', default=[],
dest='platform_repofiles')
def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None,
stream=None, skiptests=False, default_streams=None,
offline=False):
offline=False, platform_repofiles=None):
""" Performs local module build using Mock
"""
if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']:
@@ -122,6 +124,11 @@ def build_module_locally(local_build_nsvs=None, yaml_file=None, srpms=None,
with app.app_context():
conf.set_item("system", "mock")
conf.set_item("base_module_repofiles", platform_repofiles)
# Use the "local" resolver for offline module builds.
if offline:
conf.set_item("resolver", "local")
# Use our own local SQLite3 database.
confdir = os.path.abspath(os.getcwd())

View File

@@ -190,7 +190,12 @@ class DBResolver(GenericResolver):
def get_module_build_dependencies(self, name=None, stream=None, version=None, context=None,
mmd=None, strict=False):
"""
Returns a dictionary of koji_tag:mmd of all the dependencies
Returns a dictionary of koji_tag:[mmd, ...] of all the dependencies of input module.
Although it is expected that single Koji tag always contain just single module build,
it does not have to be a true for Offline local builds which use the local repository
identifier as `koji_tag`.
:kwarg name: a string of a module's name (required if mmd is not set)
:kwarg stream: a string of a module's stream (required if mmd is not set)
:kwarg version: a string of a module's version (required if mmd is not set)
@@ -240,7 +245,8 @@ class DBResolver(GenericResolver):
if not build:
raise RuntimeError(
'Buildrequired module %s %r does not exist in MBS db' % (br_name, details))
module_tags[build.koji_tag] = build.mmd()
module_tags.setdefault(build.koji_tag, [])
module_tags[build.koji_tag].append(build.mmd())
return module_tags

View File

@@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Jan Kaluza <jkaluza@redhat.com>
from module_build_service.resolver.DBResolver import DBResolver
class LocalResolver(DBResolver):
"""
Resolver using DNF and local repositories.
It is subclass of DBResolver with small changes to DBResolver logic to fit
the offline local module builds. See particular methods for more information.
"""
backend = 'local'
def get_buildrequired_modulemds(self, name, stream, base_module_nsvc):
"""
Returns modulemd metadata of all module builds with `name` and `stream`.
For LocalResolver which is used only for Offline local builds,
the `base_module_nsvc` is ignored. Normally, the `base_module_nsvc is used
to filter out platform:streams which are not compatible with currently used
stream version. But during offline local builds, we always have just single
platform:stream derived from PLATFORM_ID in /etc/os-release.
Because we have just single platform stream, there is no reason to filter
incompatible streams. This platform stream is also expected to not follow
the "X.Y.Z" formatting which is needed for stream versions.
:param str name: Name of module to return.
:param str stream: Stream of module to return.
:param str base_module_nsvc: Ignored in LocalResolver.
:rtype: list
:return: List of modulemd metadata.
"""
return self.get_module_modulemds(name, stream)

View File

@@ -242,7 +242,12 @@ class MBSResolver(GenericResolver):
def get_module_build_dependencies(self, name=None, stream=None, version=None, context=None,
mmd=None, strict=False):
"""Get module's build dependencies defined in xmd/mbs section
"""
Returns a dictionary of koji_tag:[mmd, ...] of all the dependencies of input module.
Although it is expected that single Koji tag always contain just single module build,
it does not have to be a true for Offline local builds which use the local repository
identifier as `koji_tag`.
:param name: a module's name (required if mmd is not set)
:param stream: a module's stream (required if mmd is not set)
@@ -252,7 +257,7 @@ class MBSResolver(GenericResolver):
:param strict: Normally this function returns None if no module can be
found. If strict=True, then an UnprocessableEntity is raised.
:return: a mapping containing buildrequire modules info in key/value pairs,
where key is koji_tag and value is the Modulemd.Module object.
where key is koji_tag and value is list of Modulemd.Module objects.
:rtype: dict(str, :class:`Modulemd.Module`)
"""
@@ -300,7 +305,8 @@ class MBSResolver(GenericResolver):
for m in modules:
if m["koji_tag"] in module_tags:
continue
module_tags[m["koji_tag"]] = self.extract_modulemd(m["modulemd"])
module_tags.setdefault(m["koji_tag"], [])
module_tags[m["koji_tag"]].append(self.extract_modulemd(m["modulemd"]))
return module_tags

View File

@@ -152,7 +152,9 @@ def init(config, session, msg):
try:
mmd = build.mmd()
record_component_builds(mmd, build, session=session)
handle_stream_collision_modules(mmd)
# The ursine.handle_stream_collision_modules is Koji specific.
if conf.system in ['koji', 'test']:
handle_stream_collision_modules(mmd)
mmd = record_filtered_rpms(mmd)
build.modulemd = to_text_type(mmd.dumps())
build.transition(conf, models.BUILD_STATES["wait"])
@@ -232,7 +234,7 @@ def get_content_generator_build_koji_tag(module_deps):
# Find out the name of Koji tag to which the module's Content
# Generator build should be tagged once the build finishes.
module_names_streams = {
mmd.get_name(): mmd.get_stream() for mmd in module_deps.values()
mmd.get_name(): mmd.get_stream() for mmds in module_deps.values() for mmd in mmds
}
for base_module_name in conf.base_module_names:
if base_module_name in module_names_streams:
@@ -315,10 +317,9 @@ def wait(config, session, msg):
builder = module_build_service.builder.GenericBuilder.create_from_module(
session, build, config)
dep_koji_tags = build_deps.keys()
log.debug("Adding dependencies %s into buildroot for module %s:%s:%s",
dep_koji_tags, build.name, build.stream, build.version)
builder.buildroot_add_repos(dep_koji_tags)
build_deps.keys(), build.name, build.stream, build.version)
builder.buildroot_add_repos(build_deps)
if not build.component_builds:
log.info("There are no components in module %r, skipping build" % build)

View File

@@ -443,7 +443,9 @@ def import_fake_base_module(nsvc):
xmd_mbs['requires'] = {}
xmd_mbs['commit'] = 'ref_%s' % context
xmd_mbs['mse'] = 'true'
xmd_mbs['koji_tag'] = 'local_build'
# Use empty "repofile://" URI for base module. The base module will use the
# `conf.base_module_names` list as list of default repositories.
xmd_mbs['koji_tag'] = 'repofile://'
mmd.set_xmd(glib.dict_values(xmd))
with models.make_session(conf) as session:
@@ -476,7 +478,7 @@ def import_builds_from_local_dnf_repos():
mmds = Modulemd.Module.new_all_from_string(mmd_data)
for mmd in mmds:
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd["mbs"]["koji_tag"] = "local_module"
xmd["mbs"]["koji_tag"] = "repofile://" + repo.repofile
xmd["mbs"]["mse"] = True
mmd.set_xmd(glib.dict_values(xmd))

View File

@@ -57,6 +57,7 @@ setup(name='module-build-service',
'mbs.resolver_backends': [
'mbs = module_build_service.resolver.MBSResolver:MBSResolver',
'db = module_build_service.resolver.DBResolver:DBResolver',
'local = module_build_service.resolver.LocalResolver:LocalResolver',
],
},
scripts=['client/mbs-cli'],

View File

@@ -8,11 +8,12 @@ from module_build_service.utils import to_text_type
import kobo.rpmlib
from module_build_service import conf
from module_build_service import conf, db
from module_build_service.models import ModuleBuild, ComponentBuild, make_session
from module_build_service.builder.MockModuleBuilder import MockModuleBuilder
from module_build_service import glib, Modulemd
from tests import clean_database
from module_build_service.utils import import_fake_base_module
from tests import clean_database, make_module
class TestMockModuleBuilder:
@@ -176,3 +177,46 @@ class TestMockModuleBuilder:
with open(os.path.join(self.resultdir, "pkglist"), "r") as fd:
pkglist = fd.read().strip()
assert not pkglist
class TestMockModuleBuilderAddRepos:
def setup_method(self, test_method):
clean_database(add_platform_module=False)
import_fake_base_module("platform:f29:1:000000")
self.platform = ModuleBuild.get_last_build_in_stream(db.session, "platform", "f29")
self.foo = make_module("foo:1:1:1", {"platform": ["f29"]}, {"platform": ["f29"]})
self.app = make_module("app:1:1:1", {"platform": ["f29"]}, {"platform": ["f29"]})
@mock.patch("module_build_service.conf.system", new="mock")
@mock.patch(
'module_build_service.config.Config.base_module_repofiles',
new_callable=mock.PropertyMock,
return_value=["/etc/yum.repos.d/bar.repo", "/etc/yum.repos.d/bar-updates.repo"],
create=True)
@mock.patch("module_build_service.builder.MockModuleBuilder.open", create=True)
@mock.patch(
"module_build_service.builder.MockModuleBuilder.MockModuleBuilder._load_mock_config")
@mock.patch(
"module_build_service.builder.MockModuleBuilder.MockModuleBuilder._write_mock_config")
def test_buildroot_add_repos(self, write_config, load_config, patched_open,
base_module_repofiles):
patched_open.side_effect = [
mock.mock_open(read_data="[fake]\nrepofile 1\n").return_value,
mock.mock_open(read_data="[fake]\nrepofile 2\n").return_value,
mock.mock_open(read_data="[fake]\nrepofile 3\n").return_value]
builder = MockModuleBuilder("user", self.app, conf, "module-app", [])
dependencies = {
"repofile://": [self.platform.mmd()],
"repofile:///etc/yum.repos.d/foo.repo": [self.foo.mmd(), self.app.mmd()]
}
builder.buildroot_add_repos(dependencies)
assert "repofile 1" in builder.yum_conf
assert "repofile 2" in builder.yum_conf
assert "repofile 3" in builder.yum_conf
assert set(builder.enabled_modules) == set(["foo:1", "app:1"])

View File

@@ -0,0 +1,76 @@
# Copyright (c) 2019 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Jan Kaluza <jkaluza@redhat.com>
import os
from datetime import datetime
from module_build_service.utils import to_text_type
import module_build_service.resolver as mbs_resolver
from module_build_service import db
from module_build_service.utils import import_mmd, load_mmd
from module_build_service.models import ModuleBuild
import tests
base_dir = os.path.join(os.path.dirname(__file__), "..")
class TestLocalResolverModule:
def setup_method(self):
tests.reuse_component_init_data()
def test_get_buildrequired_modulemds(self):
mmd = load_mmd(os.path.join(base_dir, 'staged_data', 'platform.yaml'), True)
mmd.set_stream('f8')
import_mmd(db.session, mmd)
platform_f8 = ModuleBuild.query.filter_by(stream='f8').one()
mmd.set_name("testmodule")
mmd.set_stream("master")
mmd.set_version(20170109091357)
mmd.set_context("123")
build = ModuleBuild(
name='testmodule',
stream='master',
version=20170109091357,
state=5,
build_context='dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3',
runtime_context='ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7',
context='7c29193d',
koji_tag='module-testmodule-master-20170109091357-7c29193d',
scmurl='https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79',
batch=3,
owner='Dr. Pepper',
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy='changed-and-after',
modulemd=to_text_type(mmd.dumps())
)
db.session.add(build)
db.session.commit()
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='local')
result = resolver.get_buildrequired_modulemds(
"testmodule", "master", platform_f8.mmd().dup_nsvc())
nsvcs = set([m.dup_nsvc() for m in result])
assert nsvcs == set(['testmodule:master:20170109091357:9c690d0e',
'testmodule:master:20170109091357:123'])

View File

@@ -191,7 +191,7 @@ class TestModuleWait:
resolver.backend = 'db'
resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357"
resolver.get_module_build_dependencies.return_value = {
"module-bootstrap-tag": base_mmd}
"module-bootstrap-tag": [base_mmd]}
with patch.object(module_build_service.resolver, 'system_resolver', new=resolver):
msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=2,
@@ -239,7 +239,7 @@ class TestModuleWait:
resolver.backend = 'db'
resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357"
resolver.get_module_build_dependencies.return_value = {
"module-bootstrap-tag": base_mmd}
"module-bootstrap-tag": [base_mmd]}
with patch.object(module_build_service.scheduler.handlers.modules.conf,
'koji_cg_tag_build', new=koji_cg_tag_build):

View File

@@ -1247,7 +1247,7 @@ class TestOfflineLocalBuilds:
'mbs': {
'buildrequires': {},
'commit': 'ref_000000',
'koji_tag': 'local_build',
'koji_tag': 'repofile://',
'mse': 'true',
'requires': {}}}
@@ -1260,6 +1260,7 @@ class TestOfflineLocalBuilds:
with patch("dnf.Base") as dnf_base:
repo = mock.MagicMock()
repo.repofile = "/etc/yum.repos.d/foo.repo"
with open(path.join(BASE_DIR, '..', 'staged_data', 'formatted_testmodule.yaml')) as f:
repo.get_metadata_content.return_value = f.read()
base = dnf_base.return_value
@@ -1277,5 +1278,8 @@ class TestOfflineLocalBuilds:
module_build = models.ModuleBuild.get_build_from_nsvc(
db.session, "testmodule", "master", 20180205135154, "9c690d0e")
assert module_build
assert module_build.koji_tag == "repofile:///etc/yum.repos.d/foo.repo"
module_build = models.ModuleBuild.get_build_from_nsvc(
db.session, "platform", "x", 1, "000000")
assert module_build