modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 11:48:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Get the RPM license headers from Koji and use it to fill the MMD content licenses field.

Browse Source
This commit is contained in:
Jan Kaluza
2018-09-11 10:05:55 +02:00
parent 0d66adbc17
commit 410f65ac3c
2 changed files with 72 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
module_build_service/builder/KojiContentGenerator.py
View File

@@ -215,36 +215,46 @@ class KojiContentGenerator(object):
# If the tag doesn't exist.. then there are no rpms in that tag.
return []
# Get the exclusivearch and excludearch lists for each RPM.
# Get the exclusivearch, excludearch and license data for each RPM.
# The exclusivearch and excludearch lists are set in source RPM from which the RPM
# was built.
# Create temporary dict with source RPMs in rpm_id:build_id format.
src_rpms_ids = {rpm["id"]: rpm["build_id"] for rpm in rpms if rpm["arch"] == "src"}
# Create temporary dict with source RPMs in rpm_id:rpms_list_index format.
src_rpms = {}
binary_rpms = {}
for rpm in rpms:
if rpm["arch"] == "src":
src_rpms[rpm["id"]] = rpm
else:
binary_rpms[rpm["id"]] = rpm
# Prepare the arguments for Koji multicall.
# We will call session.getRPMHeaders(...) for each SRC RPM to get exclusivearch and
# excludearch headers.
multicall_kwargs = [{"rpmID": rpm_id, "headers": ["exclusivearch", "excludearch"]}
for rpm_id in src_rpms_ids.keys()]
src_rpms_headers = koji_retrying_multicall_map(
# We will call session.getRPMHeaders(...) for each SRC RPM to get exclusivearch,
# excludearch and license headers.
multicall_kwargs = [{"rpmID": rpm_id,
"headers": ["exclusivearch", "excludearch", "license"]}
for rpm_id in src_rpms.keys()]
# For each binary RPM, we only care about the "license" header.
multicall_kwargs += [{"rpmID": rpm_id, "headers": ["license"]}
for rpm_id in binary_rpms.keys()]
rpms_headers = koji_retrying_multicall_map(
session, session.getRPMHeaders, list_of_kwargs=multicall_kwargs)
# Temporary dict with build_id as a key to find builds easily.
builds = {build['build_id']: build for build in builds}
# Handle the multicall result. For each build associated with the source RPM,
# store the exclusivearch and excludearch lists.
for build_id, headers in zip(src_rpms_ids.values(), src_rpms_headers):
builds[build_id]["exclusivearch"] = headers["exclusivearch"]
builds[build_id]["excludearch"] = headers["excludearch"]
# store the exclusivearch and excludearch lists. For each RPM, store the 'license' and
# also other useful data from the Build associated with the RPM.
for rpm, headers in zip(src_rpms.values() + binary_rpms.values(), rpms_headers):
build = builds[rpm["build_id"]]
if "exclusivearch" in headers and "excludearch" in headers:
build["exclusivearch"] = headers["exclusivearch"]
build["excludearch"] = headers["excludearch"]
# Check each RPM and fill-in additional data from its build to get them
# easily in other methods.
for rpm in rpms:
idx = rpm['build_id']
rpm['srpm_name'] = builds[idx]['name']
rpm['srpm_nevra'] = builds[idx]['nvr']
rpm['exclusivearch'] = builds[idx]['exclusivearch']
rpm['excludearch'] = builds[idx]['excludearch']
rpm["license"] = headers["license"]
rpm['srpm_name'] = build['name']
rpm['srpm_nevra'] = build['nvr']
rpm['exclusivearch'] = build['exclusivearch']
rpm['excludearch'] = build['excludearch']
return rpms
@@ -422,7 +432,7 @@ class KojiContentGenerator(object):
def _fill_in_rpms_list(self, mmd, arch):
"""
Fills in the list of built RPMs in architecture specific `mmd` for `arch`
using the data from `self.rpms_dict`.
using the data from `self.rpms_dict` as well as the content licenses field.
:param Modulemd.Module mmd: MMD to add built RPMs to.
:param str arch: Architecture for which to add RPMs.
@@ -445,6 +455,9 @@ class KojiContentGenerator(object):
# Modulemd.SimpleSet into which we will add the RPMs.
rpm_artifacts = Modulemd.SimpleSet()
# Modulemd.SimpleSet into which we will add licenses of all RPMs.
rpm_licenses = Modulemd.SimpleSet()
# Check each RPM in `self.rpms_dict` to find out if it can be included in mmd
# for this architecture.
for nevra, rpm in self.rpms_dict.items():
@@ -515,6 +528,11 @@ class KojiContentGenerator(object):
# Add RPM to packages.
rpm_artifacts.add(nevra)
# Not all RPMs have licenses (for example debuginfo packages).
if "license" in rpm and rpm["license"]:
rpm_licenses.add(rpm["license"])
mmd.set_content_licenses(rpm_licenses)
mmd.set_rpm_artifacts(rpm_artifacts)
return mmd
@@ -522,7 +540,6 @@ class KojiContentGenerator(object):
"""
Finalizes the modulemd:
- Fills in the list of built RPMs respecting filters, whitelist and multilib.
- TODO: Fills in the list of licences.
:param str arch: Name of arch to generate the final modulemd for.
:rtype: str
@@ -532,7 +549,6 @@ class KojiContentGenerator(object):
# Fill in the list of built RPMs.
mmd = self._fill_in_rpms_list(mmd, arch)
# TODO: Fill in the licences.
return unicode(mmd.dumps())
def _prepare_file_directory(self):

36
tests/test_content_generator.py
View File

@@ -19,7 +19,9 @@
# SOFTWARE.
#
# Written by Stanislav Ochotnicky <sochotnicky@redhat.com>
# Jan Kaluza <jkaluza@redhat.com>
import pytest
import json
import os
@@ -364,8 +366,10 @@ class TestBuild:
koji_session.listTaggedRPMS.return_value = (rpms, builds)
koji_session.multiCall.side_effect = [
# getRPMHeaders response
[[{'excludearch': ["x86_64"], 'exclusivearch': []}],
[{'excludearch': [], 'exclusivearch': ["x86_64"]}]]
[[{'excludearch': ["x86_64"], 'exclusivearch': [], 'license': 'MIT'}],
[{'excludearch': [], 'exclusivearch': ["x86_64"], 'license': 'GPL'}],
[{'license': 'MIT'}],
[{'license': 'GPL'}]]
]
get_session.return_value = koji_session
@@ -374,11 +378,14 @@ class TestBuild:
# We want to mainly check the excludearch and exclusivearch code.
if rpm["name"] == "module-build-macros":
assert rpm["excludearch"] == ["x86_64"]
assert rpm["license"] == "MIT"
else:
assert rpm["exclusivearch"] == ["x86_64"]
assert rpm["license"] == "GPL"
def _add_test_rpm(self, nevra, srpm_name=None, multilib=None,
koji_srpm_name=None, excludearch=None, exclusivearch=None):
koji_srpm_name=None, excludearch=None, exclusivearch=None,
license=None):
"""
Helper method to add test RPM to ModuleBuild used by KojiContentGenerator
and also to Koji tag used to generate the Content Generator build.
@@ -392,6 +399,7 @@ class TestBuild:
`srpm_name` is "httpd" but `koji_srpm_name` would be "httpd24-httpd".
:param list excludearch: List of architectures this package is excluded from.
:param list exclusivearch: List of architectures this package is exclusive for.
:param str license: License of this RPM.
"""
parsed_nevra = kobo.rpmlib.parse_nvra(nevra)
parsed_nevra["payloadhash"] = "hash"
@@ -401,6 +409,7 @@ class TestBuild:
parsed_nevra["srpm_name"] = srpm_name
parsed_nevra["excludearch"] = excludearch or []
parsed_nevra["exclusivearch"] = exclusivearch or []
parsed_nevra["license"] = license or ""
self.cg.rpms.append(parsed_nevra)
self.cg.rpms_dict[nevra] = parsed_nevra
@@ -519,3 +528,24 @@ class TestBuild:
"dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",
"dhcp-libs-12:4.3.5-5.module_2118aef6.i686",
"perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])
@pytest.mark.parametrize(
"licenses, expected", (
(["GPL", "MIT"], ["GPL", "MIT"]),
(["GPL", ""], ["GPL"]),
(["GPL", "GPL"], ["GPL"]),
)
)
def test_fill_in_rpms_list_license(self, licenses, expected):
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",
license=licenses[0])
self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine",
license=licenses[1])
self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine")
mmd = self.cg.module.mmd()
mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")
# Only x86_64 packages should be filled in, because we requested x86_64 arch.
assert set(mmd.get_content_licenses().get()) == set(expected)