From 5eb4c1652fdefeca87a92cd8ffee0b2ad558b603 Mon Sep 17 00:00:00 2001 From: Stanislav Ochotnicky Date: Tue, 9 May 2017 15:35:04 +0200 Subject: [PATCH] Modify content generator based on Koji dev feedback - Change the type of build from modulemd to just module - Change build output type from modulemd to file - Provide the typeinfo also for the modulemd.yaml output - Convert dashes to underscores for build version (i.e. mbs stream). koji build versions can't have dashes - we can provide real value in the extra section - Add name, stream, version data into build extra section - Add tool information for modulemd - Add buildroot components from host running mbs - Added few methods licensed under BSD 3-clause (from atomic-reactor) --- README.rst | 8 + .../builder/KojiContentGenerator.py | 142 ++++++++++++++++-- tests/test_content_generator.py | 10 +- ...st_get_generator_json_expected_output.json | 42 +++++- 4 files changed, 188 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index e95690fd..3b092e00 100644 --- a/README.rst +++ b/README.rst @@ -484,3 +484,11 @@ the following rules (all of them are evaluated from top to bottom): - if `MODULE_BUILD_SERVICE_DEVELOPER_ENV` is set to some reasonable value, DevConfiguration is forced and `config.py` is used directly from the MBS's develop instance. For more information see `docs/CONTRIBUTING.rst`. + +License +======= + +MBS is licensed under MIT license. See LICENSE file for details. + +Parts of MBS are licensed under 3-clause BSD license from: +https://github.com/projectatomic/atomic-reactor/blob/master/LICENSE diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 582f7f2c..118dec67 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -27,8 +27,10 @@ import hashlib import logging import json import os +import pkg_resources import platform import shutil +import subprocess import tempfile import koji @@ -59,6 +61,122 @@ class KojiContentGenerator(object): def __repr__(self): return "" % (self.module_name) + @staticmethod + def parse_rpm_output(output, tags, separator=';'): + """ + Copied from https://github.com/projectatomic/atomic-reactor/blob/master/atomic_reactor/plugins/exit_koji_promote.py + License: BSD 3-clause + + Parse output of the rpm query. + :param output: list, decoded output (str) from the rpm subprocess + :param tags: list, str fields used for query output + :return: list, dicts describing each rpm package + """ + + def field(tag): + """ + Get a field value by name + """ + try: + value = fields[tags.index(tag)] + except ValueError: + return None + + if value == '(none)': + return None + + return value + + components = [] + sigmarker = 'Key ID ' + for rpm in output: + fields = rpm.rstrip('\n').split(separator) + if len(fields) < len(tags): + continue + + signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig') + if signature: + parts = signature.split(sigmarker, 1) + if len(parts) > 1: + signature = parts[1] + + component_rpm = { + 'type': 'rpm', + 'name': field('NAME'), + 'version': field('VERSION'), + 'release': field('RELEASE'), + 'arch': field('ARCH'), + 'sigmd5': field('SIGMD5'), + 'signature': signature, + } + + # Special handling for epoch as it must be an integer or None + epoch = field('EPOCH') + if epoch is not None: + epoch = int(epoch) + + component_rpm['epoch'] = epoch + + if component_rpm['name'] != 'gpg-pubkey': + components.append(component_rpm) + + return components + + def __get_rpms(self): + """ + Copied from https://github.com/projectatomic/atomic-reactor/blob/master/atomic_reactor/plugins/exit_koji_promote.py + License: BSD 3-clause + + Build a list of installed RPMs in the format required for the + metadata. + """ + + tags = [ + 'NAME', + 'VERSION', + 'RELEASE', + 'ARCH', + 'EPOCH', + 'SIGMD5', + 'SIGPGP:pgpsig', + 'SIGGPG:pgpsig', + ] + + sep = ';' + fmt = sep.join(["%%{%s}" % tag for tag in tags]) + cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt) + try: + # py3 + (status, output) = subprocess.getstatusoutput(cmd) + except AttributeError: + # py2 + with open('/dev/null', 'r+') as devnull: + p = subprocess.Popen(cmd, + shell=True, + stdin=devnull, + stdout=subprocess.PIPE, + stderr=devnull) + + (stdout, stderr) = p.communicate() + status = p.wait() + output = stdout.decode() + + if status != 0: + log.debug("%s: stderr output: %s", cmd, stderr) + raise RuntimeError("%s: exit code %s" % (cmd, status)) + + return self.parse_rpm_output(output.splitlines(), tags, separator=sep) + + def __get_tools(self): + """Return list of tools which are important for reproducing mbs outputs""" + + tools = ["modulemd"] + ret = [] + for tool in tools: + ret.append({"name": tool, + "version": pkg_resources.get_distribution(tool).version}) + return ret + def _koji_rpms_in_tag(self, tag): """ Return the list of koji rpms in a tag. """ log.debug("Listing rpms in koji tag %s", tag) @@ -83,7 +201,7 @@ class KojiContentGenerator(object): def _get_build(self): ret = {} ret['name'] = self.module.name - ret['version'] = self.module.stream + ret['version'] = self.module.stream.replace("-", "_") ret['release'] = self.module.version ret['source'] = self.module.scmurl ret['start_time'] = calendar.timegm( @@ -92,16 +210,18 @@ class KojiContentGenerator(object): self.module.time_completed.utctimetuple()) ret['extra'] = { "typeinfo": { - "modulemd": { + "module": { "module_build_service_id": self.module.id, - "modulemd_str": self.module.modulemd + "modulemd_str": self.module.modulemd, + "name": self.module.name, + "stream": self.module.stream, + "version": self.module.version } } } return ret def _get_buildroot(self): - import pkg_resources version = pkg_resources.get_distribution("module-build-service").version distro = platform.linux_distribution() ret = { @@ -118,13 +238,12 @@ class KojiContentGenerator(object): "arch": platform.machine(), "type": "none" }, - "components": [], - "tools": [] + "components": self.__get_rpms(), + "tools": self.__get_tools() } return ret - def _get_output(self): ret = [] rpms = self._koji_rpms_in_tag(self.module.koji_tag) @@ -145,8 +264,13 @@ class KojiContentGenerator(object): ret.append( { 'buildroot_id': 1, - 'arch': "noarch", - 'type': 'modulemd', + 'arch': 'noarch', + 'type': 'file', + 'extra': { + 'typeinfo': { + 'module': {} + } + }, 'filesize': len(self.mmd), 'checksum_type': 'md5', 'checksum': hashlib.md5(self.mmd).hexdigest(), diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index 23e372f9..b32d7ca7 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -69,17 +69,25 @@ class TestBuild(unittest.TestCase): import moksha.hub.reactor self.vcr.__exit__() + @patch("subprocess.Popen") @patch("pkg_resources.get_distribution") @patch("platform.linux_distribution") @patch("platform.machine") @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag") - def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res): + def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, popen): """ Test generation of content generator json """ self.maxDiff = None distro.return_value = ("Fedora", "25", "Twenty Five") machine.return_value = "i686" pkg_res.return_value = Mock() pkg_res.return_value.version = "current-tested-version" + rpm_mock = Mock() + rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \ + "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2" + attrs = {'communicate.return_value': (rpm_out, 'error'), + 'wait.return_value': 0} + rpm_mock.configure_mock(**attrs) + popen.return_value = rpm_mock tests_dir = path.abspath(path.dirname(__file__)) rpm_in_tag_path = path.join(tests_dir, diff --git a/tests/test_get_generator_json_expected_output.json b/tests/test_get_generator_json_expected_output.json index 981e59c6..de3aa3f8 100644 --- a/tests/test_get_generator_json_expected_output.json +++ b/tests/test_get_generator_json_expected_output.json @@ -9,8 +9,34 @@ "name": "module-build-service", "version": "current-tested-version" }, - "tools": [], - "components": [], + "tools": [ + { + "name": "modulemd", + "version": "current-tested-version" + } + ], + "components": [ + { + "name": "rpm-name", + "version": "1.0", + "release": "r1", + "epoch": null, + "arch": "x86_64", + "sigmd5": "sigmd5:1", + "signature": "sigpgp:p", + "type": "rpm" + }, + { + "name": "rpm-name-2", + "version": "2.0", + "release": "r2", + "epoch": 1, + "arch": "i686", + "sigmd5": "sigmd5:2", + "signature": "sigpgp:p2", + "type": "rpm" + } + ], "container": { "arch": "i686", "type": "none" @@ -602,7 +628,12 @@ "filesize": 1134, "checksum": "bf1615b15f6a0fee485abe94af6b56b6", "checksum_type": "md5", - "type": "modulemd" + "type": "file", + "extra": { + "typeinfo": { + "module": {} + } + } } ], "metadata_version": 0, @@ -613,7 +644,10 @@ "release": "2", "extra": { "typeinfo": { - "modulemd": { + "module": { + "name": "nginx", + "stream": "1", + "version": "2", "module_build_service_id": 1, "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n" }