mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-04-03 18:59:30 +08:00
Merge #1237 [v2] Fix -debuginfo/-debugsource packages handling in KojiContentGenerator.
This commit is contained in:
Jan Kaluža
@@ -34,7 +34,6 @@ import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from io import open
|
||||
from collections import defaultdict
|
||||
from itertools import chain
|
||||
|
||||
import kobo.rpmlib
|
||||
@@ -564,25 +563,46 @@ class KojiContentGenerator(object):
|
||||
# Modulemd.SimpleSet into which we will add licenses of all RPMs.
|
||||
rpm_licenses = Modulemd.SimpleSet()
|
||||
|
||||
# RPM name suffixes that imply relationship to another RPM
|
||||
group_suffixes = ("-debuginfo", "-debugsource")
|
||||
# RPM name suffixes for debug RPMs.
|
||||
debug_suffixes = ("-debuginfo", "-debugsource")
|
||||
|
||||
# Grouping of "main" RPM with the corresponding debuginfo and debugsource RPMs
|
||||
grouped_rpms = defaultdict(list)
|
||||
source_rpms = {}
|
||||
# The Name:NEVRA of source RPMs which are included in final MMD.
|
||||
non_devel_source_rpms = {}
|
||||
for nevra, rpm in self.rpms_dict.items():
|
||||
name = strip_suffixes(rpm["name"], group_suffixes)
|
||||
grouped_rpms[name].append((nevra, rpm))
|
||||
if rpm["arch"] == "src":
|
||||
source_rpms[name] = nevra
|
||||
|
||||
# Check each RPM in `self.rpms_dict` to find out if it can be included in mmd
|
||||
# for this architecture.
|
||||
# Map source RPM name to NEVRA.
|
||||
source_rpms = {}
|
||||
|
||||
# Names of binary RPMs in the Koji tag.
|
||||
binary_rpm_names = set()
|
||||
|
||||
# Names of binary RPMs for which the `self._should_include()` method returned True.
|
||||
included_rpm_names = set()
|
||||
|
||||
# Names source RPMs which have some RPM built from this SRPM included in a final MMD.
|
||||
included_srpm_names = set()
|
||||
|
||||
# We need to evaluate the non-debug RPMs at first to find out which are included
|
||||
# in the final MMD and then decide whether to include the debug RPMs based on that.
|
||||
# In order to do that, we need to group debug RPMs and non-debug RPMs.
|
||||
# We also fill in the `binary_rpm_names` and `source_rpms` here.
|
||||
debug_rpms = {}
|
||||
non_debug_rpms = {}
|
||||
for nevra, rpm in self.rpms_dict.items():
|
||||
# Filter out debug and source RPMs, these will later be included if
|
||||
# the "main" RPM is included
|
||||
if rpm["name"].endswith(group_suffixes) or rpm["arch"] == "src":
|
||||
if rpm["arch"] == "src":
|
||||
source_rpms[rpm["name"]] = nevra
|
||||
else:
|
||||
binary_rpm_names.add(rpm["name"])
|
||||
if rpm["name"].endswith(debug_suffixes):
|
||||
debug_rpms[nevra] = rpm
|
||||
else:
|
||||
non_debug_rpms[nevra] = rpm
|
||||
|
||||
# Check each RPM in Koji tag to find out if it can be included in mmd
|
||||
# for this architecture.
|
||||
for nevra, rpm in chain(non_debug_rpms.items(), debug_rpms.items()):
|
||||
# Filter out source RPMs, these will later be included if
|
||||
# the "main" RPM is included.
|
||||
if rpm["arch"] == "src":
|
||||
continue
|
||||
|
||||
# Filter out RPMs which will never end up in final modulemd:
|
||||
@@ -599,7 +619,27 @@ class KojiContentGenerator(object):
|
||||
if rpm["exclusivearch"] and not set(rpm["exclusivearch"]) & set(exclusive_arches):
|
||||
continue
|
||||
|
||||
should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches)
|
||||
# The debug RPMs are handled differently ...
|
||||
if rpm["name"].endswith(debug_suffixes):
|
||||
# We include foo-debuginfo/foo-debugsource RPMs only in one of these cases:
|
||||
# - The "foo" is included in a MMD file (it means it is not filtered out).
|
||||
# - The "foo" package does not exist at all (it means only foo-debuginfo exists
|
||||
# and we need to include this package unless filtered out) and in the same time
|
||||
# the SRPM from which this -debuginfo/-debugsource RPM has been built is included
|
||||
# in a final MMD (it means that there is at least some package from this build
|
||||
# included - this handles case when only foo.src.rpm and foo-debugsource.rpm
|
||||
# would be included in the final MMD, which would be wrong.)
|
||||
# We also respect filters here, so it is possible to explicitely filter out also
|
||||
# -debuginfo/-debugsource packages.
|
||||
main_rpm_name = strip_suffixes(rpm["name"], debug_suffixes)
|
||||
if (main_rpm_name in included_rpm_names or (
|
||||
main_rpm_name not in binary_rpm_names
|
||||
and rpm["srpm_name"] in included_srpm_names)):
|
||||
should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches)
|
||||
else:
|
||||
should_include = False
|
||||
else:
|
||||
should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches)
|
||||
|
||||
# A source RPM should be included in a -devel module only if all the
|
||||
# RPMs built from this source RPM are included in a -devel module.
|
||||
@@ -608,6 +648,8 @@ class KojiContentGenerator(object):
|
||||
# list for -devel modules.
|
||||
if should_include:
|
||||
non_devel_source_rpms[rpm["name"]] = rpm["srpm_nevra"]
|
||||
included_rpm_names.add(rpm["name"])
|
||||
included_srpm_names.add(rpm["srpm_name"])
|
||||
|
||||
if self.devel and should_include:
|
||||
# In case this is a -devel module, we want to skip any RPMs which would normally be
|
||||
@@ -619,16 +661,11 @@ class KojiContentGenerator(object):
|
||||
# really should include and skip the others.
|
||||
continue
|
||||
|
||||
# Add RPM and its related RPMs to packages.
|
||||
for related_nevra, related_rpm in grouped_rpms[rpm["name"]]:
|
||||
if related_rpm["arch"] != rpm["arch"]:
|
||||
continue
|
||||
|
||||
rpm_artifacts.add(related_nevra)
|
||||
# Not all RPMs have licenses (for example debuginfo packages).
|
||||
license = related_rpm.get("license")
|
||||
if license:
|
||||
rpm_licenses.add(license)
|
||||
rpm_artifacts.add(nevra)
|
||||
# Not all RPMs have licenses (for example debuginfo packages).
|
||||
license = rpm.get("license")
|
||||
if license:
|
||||
rpm_licenses.add(license)
|
||||
|
||||
if self.devel:
|
||||
for source_nevra in set(source_rpms.values()) - set(non_devel_source_rpms.values()):
|
||||
|
||||
@@ -991,3 +991,185 @@ class TestBuild:
|
||||
side_effect=koji.GenericError("Build already exists asdv"))
|
||||
self.cg.koji_import()
|
||||
tagger.assert_called()
|
||||
|
||||
def test_fill_in_rpms_list_debuginfo_deps(self):
|
||||
"""
|
||||
Tests that -debuginfo RPM required by other -debuginfo RPM is included in a RPM list.
|
||||
|
||||
The python3-pymongo has matching python3-pymongo-debuginfo RPM which requires
|
||||
python-pymongo-debuginfo RPM. All of them should appear in RPM list
|
||||
"""
|
||||
self._add_test_rpm(
|
||||
"python-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64",
|
||||
"python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python3-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64",
|
||||
"python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python3-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64",
|
||||
"python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src",
|
||||
"python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src")
|
||||
|
||||
mmd = self.cg.module.mmd()
|
||||
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
|
||||
|
||||
assert set(mmd.get_rpm_artifacts().get()) == set([
|
||||
'python-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64',
|
||||
'python3-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64',
|
||||
'python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src',
|
||||
'python3-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64'])
|
||||
|
||||
def test_fill_in_rpms_list_debuginfo_deps_psycopg2(self):
|
||||
"""
|
||||
Tests that -debuginfo RPM required by other -debuginfo RPM is included in a RPM list
|
||||
with the psycopg2 RPM test-case, because psycopg2 RPMs are built in kind of special
|
||||
way...
|
||||
"""
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-debug-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-tests-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-debug-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
mmd = self.cg.module.mmd()
|
||||
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
|
||||
|
||||
assert set(mmd.get_rpm_artifacts().get()) == set([
|
||||
"python2-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python2-psycopg2-debug-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python2-psycopg2-tests-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python2-psycopg2-debug-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src"])
|
||||
|
||||
def test_fill_in_rpms_list_debugsource_for_non_srpm(self):
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
mmd = self.cg.module.mmd()
|
||||
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
|
||||
|
||||
assert set(mmd.get_rpm_artifacts().get()) == set([
|
||||
"python2-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src"])
|
||||
|
||||
def test_fill_in_rpms_list_debuginfo_deps_glibc(self):
|
||||
self._add_test_rpm(
|
||||
"glibc-common-2.29.9000-16.fc31.x86_64",
|
||||
"glibc-2.29.9000-16.fc31.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"glibc-2.29.9000-16.fc31.x86_64",
|
||||
"glibc-2.29.9000-16.fc31.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"glibc-debuginfo-common-2.29.9000-16.fc31.x86_64",
|
||||
"glibc-2.29.9000-16.fc31.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"glibc-debuginfo-2.29.9000-16.fc31.x86_64",
|
||||
"glibc-2.29.9000-16.fc31.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"glibc-2.29.9000-16.fc31.src",
|
||||
"glibc-2.29.9000-16.fc31.src")
|
||||
|
||||
mmd = self.cg.module.mmd()
|
||||
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
|
||||
|
||||
assert set(mmd.get_rpm_artifacts().get()) == set([
|
||||
"glibc-common-2.29.9000-16.fc31.x86_64",
|
||||
"glibc-2.29.9000-16.fc31.src",
|
||||
"glibc-2.29.9000-16.fc31.x86_64",
|
||||
"glibc-debuginfo-common-2.29.9000-16.fc31.x86_64",
|
||||
"glibc-debuginfo-2.29.9000-16.fc31.x86_64"])
|
||||
|
||||
def test_fill_in_rpms_list_debuginfo_deps_kernel(self):
|
||||
self._add_test_rpm(
|
||||
"kernel-debuginfo-common-aarch64-5.0.9-301.fc30.aarch64",
|
||||
"kernel-5.0.9-301.fc30.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"kernel-debuginfo-5.0.9-301.fc30.aarch64",
|
||||
"kernel-5.0.9-301.fc30.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"kernel-5.0.9-301.fc30.aarch64",
|
||||
"kernel-5.0.9-301.fc30.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"kernel-5.0.9-301.fc30.src",
|
||||
"kernel-5.0.9-301.fc30.src")
|
||||
|
||||
mmd = self.cg.module.mmd()
|
||||
mmd = self.cg._fill_in_rpms_list(mmd, "aarch64")
|
||||
|
||||
assert set(mmd.get_rpm_artifacts().get()) == set([
|
||||
"kernel-debuginfo-common-aarch64-5.0.9-301.fc30.aarch64",
|
||||
"kernel-5.0.9-301.fc30.src",
|
||||
"kernel-debuginfo-5.0.9-301.fc30.aarch64",
|
||||
"kernel-5.0.9-301.fc30.aarch64"])
|
||||
|
||||
def test_fill_in_rpms_list_debugsource_not_included(self):
|
||||
self._add_test_rpm(
|
||||
"python-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
self._add_test_rpm(
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src",
|
||||
"python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src")
|
||||
|
||||
mmd = self.cg.module.mmd()
|
||||
filter_list = Modulemd.SimpleSet()
|
||||
filter_list.add("python2-psycopg2")
|
||||
mmd.set_rpm_filter(filter_list)
|
||||
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
|
||||
|
||||
assert set(mmd.get_rpm_artifacts().get()) == set([])
|
||||
|
||||
Reference in New Issue
Block a user