Modify content generator based on Koji dev feedback

- Change the type of build from modulemd to just module
- Change build output type from modulemd to file
- Provide the typeinfo also for the modulemd.yaml output
- Convert dashes to underscores for build version (i.e. mbs stream).
  koji build versions can't have dashes - we can provide real value in
  the extra section
- Add name, stream, version data into build extra section
- Add tool information for modulemd
- Add buildroot components from host running mbs
- Added few methods licensed under BSD 3-clause (from atomic-reactor)

README.rst

@@ -484,3 +484,11 @@ the following rules (all of them are evaluated from top to bottom):
 - if `MODULE_BUILD_SERVICE_DEVELOPER_ENV` is set to some reasonable
   value, DevConfiguration is forced and `config.py` is used directly from the
   MBS's develop instance. For more information see `docs/CONTRIBUTING.rst`.
+
+License
+=======
+
+MBS is licensed under MIT license. See LICENSE file for details.
+
+Parts of MBS are licensed under 3-clause BSD license from:
+https://github.com/projectatomic/atomic-reactor/blob/master/LICENSE

module_build_service/builder/KojiContentGenerator.py

@@ -27,8 +27,10 @@ import hashlib
 import logging
 import json
 import os
+import pkg_resources
 import platform
 import shutil
+import subprocess
 import tempfile
 
 import koji
@@ -59,6 +61,122 @@ class KojiContentGenerator(object):
     def __repr__(self):
         return "<KojiContentGenerator module: %s>" % (self.module_name)
 
+    @staticmethod
+    def parse_rpm_output(output, tags, separator=';'):
+        """
+        Copied from https://github.com/projectatomic/atomic-reactor/blob/master/atomic_reactor/plugins/exit_koji_promote.py
+        License: BSD 3-clause
+
+        Parse output of the rpm query.
+        :param output: list, decoded output (str) from the rpm subprocess
+        :param tags: list, str fields used for query output
+        :return: list, dicts describing each rpm package
+        """
+
+        def field(tag):
+            """
+            Get a field value by name
+            """
+            try:
+                value = fields[tags.index(tag)]
+            except ValueError:
+                return None
+
+            if value == '(none)':
+                return None
+
+            return value
+
+        components = []
+        sigmarker = 'Key ID '
+        for rpm in output:
+            fields = rpm.rstrip('\n').split(separator)
+            if len(fields) < len(tags):
+                continue
+
+            signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig')
+            if signature:
+                parts = signature.split(sigmarker, 1)
+                if len(parts) > 1:
+                    signature = parts[1]
+
+            component_rpm = {
+                'type': 'rpm',
+                'name': field('NAME'),
+                'version': field('VERSION'),
+                'release': field('RELEASE'),
+                'arch': field('ARCH'),
+                'sigmd5': field('SIGMD5'),
+                'signature': signature,
+            }
+
+            # Special handling for epoch as it must be an integer or None
+            epoch = field('EPOCH')
+            if epoch is not None:
+                epoch = int(epoch)
+
+            component_rpm['epoch'] = epoch
+
+            if component_rpm['name'] != 'gpg-pubkey':
+                components.append(component_rpm)
+
+        return components
+
+    def __get_rpms(self):
+        """
+        Copied from https://github.com/projectatomic/atomic-reactor/blob/master/atomic_reactor/plugins/exit_koji_promote.py
+        License: BSD 3-clause
+
+        Build a list of installed RPMs in the format required for the
+        metadata.
+        """
+
+        tags = [
+            'NAME',
+            'VERSION',
+            'RELEASE',
+            'ARCH',
+            'EPOCH',
+            'SIGMD5',
+            'SIGPGP:pgpsig',
+            'SIGGPG:pgpsig',
+        ]
+
+        sep = ';'
+        fmt = sep.join(["%%{%s}" % tag for tag in tags])
+        cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)
+        try:
+            # py3
+            (status, output) = subprocess.getstatusoutput(cmd)
+        except AttributeError:
+            # py2
+            with open('/dev/null', 'r+') as devnull:
+                p = subprocess.Popen(cmd,
+                                     shell=True,
+                                     stdin=devnull,
+                                     stdout=subprocess.PIPE,
+                                     stderr=devnull)
+
+                (stdout, stderr) = p.communicate()
+                status = p.wait()
+                output = stdout.decode()
+
+        if status != 0:
+            log.debug("%s: stderr output: %s", cmd, stderr)
+            raise RuntimeError("%s: exit code %s" % (cmd, status))
+
+        return self.parse_rpm_output(output.splitlines(), tags, separator=sep)
+
+    def __get_tools(self):
+        """Return list of tools which are important for reproducing mbs outputs"""
+
+        tools = ["modulemd"]
+        ret = []
+        for tool in tools:
+            ret.append({"name": tool,
+                        "version": pkg_resources.get_distribution(tool).version})
+        return ret
+
     def _koji_rpms_in_tag(self, tag):
         """ Return the list of koji rpms in a tag. """
         log.debug("Listing rpms in koji tag %s", tag)
@@ -83,7 +201,7 @@ class KojiContentGenerator(object):
     def _get_build(self):
         ret = {}
         ret['name'] = self.module.name
-        ret['version'] = self.module.stream
+        ret['version'] = self.module.stream.replace("-", "_")
         ret['release'] = self.module.version
         ret['source'] = self.module.scmurl
         ret['start_time'] = calendar.timegm(
@@ -92,16 +210,18 @@ class KojiContentGenerator(object):
             self.module.time_completed.utctimetuple())
         ret['extra'] = {
             "typeinfo": {
-                "modulemd": {
+                "module": {
                     "module_build_service_id": self.module.id,
-                    "modulemd_str": self.module.modulemd
+                    "modulemd_str": self.module.modulemd,
+                    "name": self.module.name,
+                    "stream": self.module.stream,
+                    "version": self.module.version
                 }
             }
         }
         return ret
 
     def _get_buildroot(self):
-        import pkg_resources
         version = pkg_resources.get_distribution("module-build-service").version
         distro = platform.linux_distribution()
         ret = {
@@ -118,13 +238,12 @@ class KojiContentGenerator(object):
                 "arch": platform.machine(),
                 "type": "none"
             },
-            "components": [],
-            "tools": []
+            "components": self.__get_rpms(),
+            "tools": self.__get_tools()
         }
         return ret
 
 
-
     def _get_output(self):
         ret = []
         rpms = self._koji_rpms_in_tag(self.module.koji_tag)
@@ -145,8 +264,13 @@ class KojiContentGenerator(object):
         ret.append(
             {
                 'buildroot_id': 1,
-                'arch': "noarch",
-                'type': 'modulemd',
+                'arch': 'noarch',
+                'type': 'file',
+                'extra': {
+                    'typeinfo': {
+                        'module': {}
+                    }
+                },
                 'filesize': len(self.mmd),
                 'checksum_type': 'md5',
                 'checksum': hashlib.md5(self.mmd).hexdigest(),

tests/test_content_generator.py

@@ -69,17 +69,25 @@ class TestBuild(unittest.TestCase):
         import moksha.hub.reactor
         self.vcr.__exit__()
 
+    @patch("subprocess.Popen")
     @patch("pkg_resources.get_distribution")
     @patch("platform.linux_distribution")
     @patch("platform.machine")
     @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag")
-    def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res):
+    def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, popen):
         """ Test generation of content generator json """
         self.maxDiff = None
         distro.return_value = ("Fedora", "25", "Twenty Five")
         machine.return_value = "i686"
         pkg_res.return_value = Mock()
         pkg_res.return_value.version = "current-tested-version"
+        rpm_mock = Mock()
+        rpm_out =  "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \
+                   "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"
+        attrs = {'communicate.return_value': (rpm_out, 'error'),
+                 'wait.return_value': 0}
+        rpm_mock.configure_mock(**attrs)
+        popen.return_value = rpm_mock
 
         tests_dir = path.abspath(path.dirname(__file__))
         rpm_in_tag_path = path.join(tests_dir,

tests/test_get_generator_json_expected_output.json

@@ -9,8 +9,34 @@
             "name": "module-build-service",
             "version": "current-tested-version"
         },
-        "tools": [],
-        "components": [],
+        "tools": [
+            {
+                "name": "modulemd",
+                "version": "current-tested-version"
+            }
+        ],
+        "components": [
+            {
+                "name": "rpm-name",
+                "version": "1.0",
+                "release": "r1",
+                "epoch": null,
+                "arch": "x86_64",
+                "sigmd5": "sigmd5:1",
+                "signature": "sigpgp:p",
+                "type": "rpm"
+            },
+            {
+                "name": "rpm-name-2",
+                "version": "2.0",
+                "release": "r2",
+                "epoch": 1,
+                "arch": "i686",
+                "sigmd5": "sigmd5:2",
+                "signature": "sigpgp:p2",
+                "type": "rpm"
+            }
+        ],
         "container": {
             "arch": "i686",
             "type": "none"
@@ -602,7 +628,12 @@
             "filesize": 1134,
             "checksum": "bf1615b15f6a0fee485abe94af6b56b6",
             "checksum_type": "md5",
-            "type": "modulemd"
+            "type": "file",
+            "extra": {
+                "typeinfo": {
+                    "module": {}
+                }
+            }
         }
     ],
     "metadata_version": 0,
@@ -613,7 +644,10 @@
         "release": "2",
         "extra": {
             "typeinfo": {
-                "modulemd": {
+                "module": {
+                    "name": "nginx",
+                    "stream": "1",
+                    "version": "2",
                     "module_build_service_id": 1,
                     "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n    # Module name, optional\n    # Typically filled in by the buildsystem, using the VCS repository\n    # name as the name of the module.\n    name: nginx\n    # Module update stream, optional\n    # Typically filled in by the buildsystem, using the VCS branch name\n    # as the name of the stream.\n    stream: 1\n    # Module version, integer, optional, cannot be negative\n    # Typically filled in by the buildsystem, using the VCS commit\n    # timestamp.  Module version defines upgrade path for the particular\n    # update stream.\n    version: 2\n    # A short summary describing the module, required\n    summary: An example nginx module\n    # A verbose description of the module, required\n    description: >\n        A module for the tests of module build service\n    # Module and content licenses in the Fedora license identifier\n    # format, required\n    license:\n        # Module license, required\n        # This list covers licenses used for the module metadata, SPEC\n        # files or extra patches\n        module:\n            - MIT\n"
                 }