modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 11:48:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Import -devel CG build with RPMs which are filtered out of the current real CG build.

Browse Source 
For some modules, modularity-wg wants to be able to ship the "-devel" modules containing
the RPMs which are normally filtered out from the module.

This PR generates second Koji CG module with -devel suffix in a name with final modulemd
files containing the filtered out RPMs.
This commit is contained in:
Jan Kaluza
2018-10-30 13:49:12 +01:00
parent 7a0360728a
commit 13ab18425a
3 changed files with 166 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
module_build_service/builder/KojiContentGenerator.py
View File

@@ -74,6 +74,7 @@ class KojiContentGenerator(object):
self.module_name = module.name
self.mmd = module.modulemd
self.config = config
self.devel = False
# List of architectures the module is built for.
self.arches = []
# List of RPMs tagged in module.koji_tag as returned by Koji.
@@ -261,6 +262,8 @@ class KojiContentGenerator(object):
def _get_build(self):
ret = {}
ret[u'name'] = self.module.name
if self.devel:
ret['name'] += "-devel"
ret[u'version'] = self.module.stream.replace("-", "_")
# Append the context to the version to make NVRs of modules unique in the event of
# module stream expansion
@@ -276,7 +279,7 @@ class KojiContentGenerator(object):
u"module_build_service_id": self.module.id,
u"content_koji_tag": self.module.koji_tag,
u"modulemd_str": self.module.modulemd,
u"name": self.module.name,
u"name": ret['name'],
u"stream": self.module.stream,
u"version": self.module.version,
u"context": self.module.context
@@ -429,6 +432,68 @@ class KojiContentGenerator(object):
return ret
def _should_include_rpm(self, rpm, mmd, arch, multilib_arches):
"""
Helper method for `_fill_in_rpms_list` returning True if the RPM object
should be included in a final MMD file for given arch.
"""
# Check the "whitelist" buildopts section of MMD.
# When "whitelist" is defined, it overrides component names in
# `mmd.get_rpm_components()`. The whitelist is used when module needs to build
# package with different SRPM name than the package name. This is case for example
# for software collections where SRPM name can be "httpd24-httpd", but package name
# is still "httpd". In this case, get_rpm_components() would contain "httpd", but the
# rpm["srpm_name"] would be "httpd24-httpd".
srpm = rpm["srpm_name"]
whitelist = None
buildopts = mmd.get_buildopts()
if buildopts:
whitelist = buildopts.get_rpm_whitelist()
if whitelist:
if srpm not in whitelist:
# Package is not in the whitelist, skip it.
return False
# If there is no whitelist, just check that the SRPM name we have here
# exists in the list of components.
# In theory, there should never be situation where modular tag contains
# some RPM built from SRPM not included in get_rpm_components() or in whitelist,
# but the original Pungi code checked for this case.
if not whitelist and srpm not in mmd.get_rpm_components().keys():
return False
# Do not include this RPM if it is filtered.
if rpm["name"] in mmd.get_rpm_filter().get():
return False
# Skip the rpm if it's built for multilib arch, but
# multilib is not enabled for this srpm in MMD.
try:
mmd_component = mmd.get_rpm_components()[srpm]
multilib = mmd_component.get_multilib()
multilib = multilib.get() if multilib else set()
# The `multilib` set defines the list of architectures for which
# the multilib is enabled.
#
# Filter out RPMs from multilib architectures if multilib is not
# enabled for current arch. Keep the RPMs from non-multilib compatible
# architectures.
if arch not in multilib and rpm["arch"] in multilib_arches:
return False
except KeyError:
# TODO: This exception is raised only when "whitelist" is used.
# Since components in whitelist have different names than ones in
# components list, we won't find them there.
# We would need to track the RPMs srpm_name from whitelist back to
# original package name used in MMD's components list. This is possible
# but original Pungi code is not doing that. This is TODO for future
# improvements.
# No such component, disable any multilib
if rpm["arch"] not in ("noarch", arch):
return False
return True
def _fill_in_rpms_list(self, mmd, arch):
"""
Fills in the list of built RPMs in architecture specific `mmd` for `arch`
@@ -473,8 +538,6 @@ class KojiContentGenerator(object):
rpm["arch"] not in [arch, "noarch", "src"]):
continue
srpm = rpm["srpm_name"]
# Skip the RPM if it is excluded on this arch or exclusive
# for different arch.
if rpm["excludearch"] and set(rpm["excludearch"]) & set(exclusive_arches):
@@ -482,61 +545,17 @@ class KojiContentGenerator(object):
if rpm["exclusivearch"] and not set(rpm["exclusivearch"]) & set(exclusive_arches):
continue
# Check the "whitelist" buildopts section of MMD.
# When "whitelist" is defined, it overrides component names in
# `mmd.get_rpm_components()`. The whitelist is used when module needs to build
# package with different SRPM name than the package name. This is case for example
# for software collections where SRPM name can be "httpd24-httpd", but package name
# is still "httpd". In this case, get_rpm_components() would contain "httpd", but the
# rpm["srpm_name"] would be "httpd24-httpd".
whitelist = None
buildopts = mmd.get_buildopts()
if buildopts:
whitelist = buildopts.get_rpm_whitelist()
if whitelist:
if srpm not in whitelist:
# Package is not in the whitelist, skip it.
continue
# If there is no whitelist, just check that the SRPM name we have here
# exists in the list of components.
# In theory, there should never be situation where modular tag contains
# some RPM built from SRPM not included in get_rpm_components() or in whitelist,
# but the original Pungi code checked for this case.
if not whitelist and srpm not in mmd.get_rpm_components().keys():
should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches)
if self.devel and should_include:
# In case this is -devel module, we want to skip any RPMs which would be normally
# include in a module and only keep those which wouldn't be included, because
# -devel is complement to normal module build.
continue
# Do not include this RPM if it is filtered.
if rpm["name"] in mmd.get_rpm_filter().get():
elif not self.devel and not should_include:
# In chase this is normal (non-devel) module, include only packages which we
# really should include and skip the others.
continue
# Skip the rpm if it's built for multilib arch, but
# multilib is not enabled for this srpm in MMD.
try:
mmd_component = mmd.get_rpm_components()[srpm]
multilib = mmd_component.get_multilib()
multilib = multilib.get() if multilib else set()
# The `multilib` set defines the list of architectures for which
# the multilib is enabled.
#
# Filter out RPMs from multilib architectures if multilib is not
# enabled for current arch. Keep the RPMs from non-multilib compatible
# architectures.
if arch not in multilib and rpm["arch"] in multilib_arches:
continue
except KeyError:
# TODO: This exception is raised only when "whitelist" is used.
# Since components in whitelist have different names than ones in
# components list, we won't find them there.
# We would need to track the RPMs srpm_name from whitelist back to
# original package name used in MMD's components list. This is possible
# but original Pungi code is not doing that. This is TODO for future
# improvements.
# No such component, disable any multilib
if rpm["arch"] not in ("noarch", arch):
continue
# Add RPM to packages.
rpm_artifacts.add(nevra)
@@ -558,6 +577,9 @@ class KojiContentGenerator(object):
:return: Finalized modulemd string.
"""
mmd = self.module.mmd()
if self.devel:
mmd.set_name(mmd.get_name() + "-devel")
# Set the "Arch" field in mmd.
mmd.set_arch(pungi.arch.tree_arch_to_yum_arch(arch))
# Fill in the list of built RPMs.
@@ -655,16 +677,29 @@ class KojiContentGenerator(object):
session.tagBuild(tag_info["id"], nvr)
def _load_koji_tag(self, koji_session):
# Do not load Koji tag data if this method is called again. This would
# waste resources, because the Koji tag content is always the same
# for already built module.
if self.arches and self.rpms and self.rpms_dict:
return
tag = koji_session.getTag(self.module.koji_tag)
self.arches = tag["arches"].split(" ") if tag["arches"] else []
self.rpms = self._koji_rpms_in_tag(self.module.koji_tag)
self.rpms_dict = {kobo.rpmlib.make_nvra(rpm, force_epoch=True): rpm for rpm in self.rpms}
def koji_import(self):
def koji_import(self, devel=False):
"""This method imports given module into the configured koji instance as
a content generator based build
Raises an exception when error is encountered during import"""
Raises an exception when error is encountered during import
:param bool devel: True if the "-devel" module should be created and imported.
The "-devel" module build contains only the RPMs which are normally filtered
from the module build. If set to False, normal module build respecting the
filters is created and imported.
"""
self.devel = devel
session = get_session(self.config, self.owner)
self._load_koji_tag(session)

1
module_build_service/builder/KojiModuleBuilder.py
View File

@@ -1176,6 +1176,7 @@ chmod 644 %buildroot/etc/rpm/macros.zz-modules
if self.config.koji_enable_content_generator and self.module.state == 3:
cg = KojiContentGenerator(self.module, self.config)
cg.koji_import()
cg.koji_import(devel=True)
@staticmethod
def get_rpm_module_tag(rpm):

103
tests/test_content_generator.py
View File

@@ -83,8 +83,9 @@ class TestBuild:
@patch("platform.machine")
@patch(("module_build_service.builder.KojiContentGenerator.KojiContentGenerator."
"_koji_rpms_in_tag"))
@pytest.mark.parametrize("devel", (False, True))
def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, coutput, popen,
get_session):
get_session, devel):
""" Test generation of content generator json """
koji_session = MagicMock()
koji_session.getUser.return_value = GET_USER_RV
@@ -117,11 +118,17 @@ class TestBuild:
build_logs.start(self.cg.module)
build_logs.stop(self.cg.module)
self.cg.devel = devel
self.cg._load_koji_tag(koji_session)
file_dir = self.cg._prepare_file_directory()
ret = self.cg._get_content_generator_metadata(file_dir)
rpms_in_tag.assert_called_once()
assert expected_output == ret
if not devel:
assert expected_output == ret
else:
# For devel, only check that the name has -devel suffix.
assert ret["build"]["name"] == "nginx-devel"
assert ret["build"]["extra"]["typeinfo"]["module"]["name"] == "nginx-devel"
@patch("module_build_service.builder.KojiContentGenerator.get_session")
@patch("subprocess.Popen")
@@ -429,7 +436,8 @@ class TestBuild:
self.cg.module.modulemd = mmd.dumps()
self.cg.modulemd = mmd.dumps()
def test_fill_in_rpms_list(self):
@pytest.mark.parametrize("devel", (False, True))
def test_fill_in_rpms_list(self, devel):
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.s390x", "dhcp")
@@ -439,15 +447,23 @@ class TestBuild:
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.s390x", "perl-Tangerine")
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.src", "perl-Tangerine")
self.cg.devel = devel
mmd = self.cg.module.mmd()
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
# Only x86_64 packages should be filled in, because we requested x86_64 arch.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",
"dhcp-libs-12:4.3.5-5.module_2118aef6.src",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.src"])
if not devel:
# Only x86_64 packages should be filled in, because we requested x86_64 arch.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",
"dhcp-libs-12:4.3.5-5.module_2118aef6.src",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.src"])
else:
# The i686 packages are filtered out in normal packages, because multilib
# is not enabled for them - therefore we want to include them in -devel.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.i686",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])
def test_fill_in_rpms_exclusivearch(self):
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",
@@ -476,7 +492,8 @@ class TestBuild:
assert set(mmd.get_rpm_artifacts().get()) == set([
"perl-Tangerine-12:4.3.5-5.module_2118aef6.noarch"])
def test_fill_in_rpms_rpm_whitelist(self):
@pytest.mark.parametrize("devel", (False, True))
def test_fill_in_rpms_rpm_whitelist(self, devel):
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",
koji_srpm_name="python27-dhcp")
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",
@@ -486,6 +503,7 @@ class TestBuild:
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",
koji_srpm_name="foo-perl-Tangerine")
self.cg.devel = devel
mmd = self.cg.module.mmd()
opts = mmd.get_buildopts()
opts.set_rpm_whitelist(["python27-dhcp"])
@@ -493,17 +511,25 @@ class TestBuild:
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
# Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted
# srpm name.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64"])
if not devel:
# Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted
# srpm name.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64"])
else:
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.i686",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])
def test_fill_in_rpms_list_filters(self):
@pytest.mark.parametrize("devel", (False, True))
def test_fill_in_rpms_list_filters(self, devel):
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine")
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine")
self.cg.devel = devel
mmd = self.cg.module.mmd()
filter_list = Modulemd.SimpleSet()
filter_list.add("dhcp-libs")
@@ -511,11 +537,18 @@ class TestBuild:
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
# Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.
assert set(mmd.get_rpm_artifacts().get()) == set([
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])
if not devel:
# Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.
assert set(mmd.get_rpm_artifacts().get()) == set([
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])
else:
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",
"dhcp-libs-12:4.3.5-5.module_2118aef6.i686",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])
def test_fill_in_rpms_list_multilib(self):
@pytest.mark.parametrize("devel", (False, True))
def test_fill_in_rpms_list_multilib(self, devel):
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",
multilib=["x86_64"])
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",
@@ -525,15 +558,20 @@ class TestBuild:
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",
multilib=["ppc64le"])
self.cg.devel = devel
mmd = self.cg.module.mmd()
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
# Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have
# multilib set.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",
"dhcp-libs-12:4.3.5-5.module_2118aef6.i686",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])
if not devel:
# Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have
# multilib set.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",
"dhcp-libs-12:4.3.5-5.module_2118aef6.i686",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])
else:
assert set(mmd.get_rpm_artifacts().get()) == set([
"perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])
@pytest.mark.parametrize(
"licenses, expected", (
@@ -556,17 +594,22 @@ class TestBuild:
# Only x86_64 packages should be filled in, because we requested x86_64 arch.
assert set(mmd.get_content_licenses().get()) == set(expected)
def test_fill_in_rpms_list_noarch_filtering_not_influenced_by_multilib(self):
@pytest.mark.parametrize("devel", (False, True))
def test_fill_in_rpms_list_noarch_filtering_not_influenced_by_multilib(self, devel):
# A build has ExcludeArch: i686 (because it only works on 64 bit arches).
# A noarch package is built there, and this noarch packages should be
# included in x86_64 repo.
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",
excludearch=["i686"])
self.cg.devel = devel
mmd = self.cg.module.mmd()
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
# Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have
# multilib set.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])
if not devel:
# Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have
# multilib set.
assert set(mmd.get_rpm_artifacts().get()) == set([
"dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])
else:
assert set(mmd.get_rpm_artifacts().get()) == set([])