Merge #528 Add functionality of koji content generator imports

module_build_service/builder/KojiContentGenerator.py

@@ -23,9 +23,13 @@
 
 
 import calendar
-import logging
-import platform
 import hashlib
+import logging
+import json
+import os
+import platform
+import shutil
+import tempfile
 
 import koji
 
@@ -47,12 +51,13 @@ class KojiContentGenerator(object):
         """
         self.owner = module.owner
         self.module = module
-        self.module_str = module.name
+        self.module_name = module.name
+        self.mmd = module.modulemd
         self.config = config
 
 
     def __repr__(self):
-        return "<KojiContentGenerator module: %s>" % (self.module_str)
+        return "<KojiContentGenerator module: %s>" % (self.module_name)
 
     def _koji_rpms_in_tag(self, tag):
         """ Return the list of koji rpms in a tag. """
@@ -142,9 +147,9 @@ class KojiContentGenerator(object):
                 'buildroot_id': 1,
                 'arch': "noarch",
                 'type': 'modulemd',
-                'filesize': len(self.module_str),
+                'filesize': len(self.mmd),
                 'checksum_type': 'md5',
-                'checksum': hashlib.md5(self.module_str).hexdigest(),
+                'checksum': hashlib.md5(self.mmd).hexdigest(),
                 'filename': 'modulemd.yaml',
                 'components': components
             }
@@ -163,7 +168,34 @@ class KojiContentGenerator(object):
 
         return ret
 
+    def _prepare_file_directory(self):
+        """ Creates a temporary directory that will contain all the files
+        mentioned in the outputs section
+
+        Returns path to the temporary directory
+        """
+        prepdir = tempfile.mkdtemp(prefix="koji-cg-import")
+        mmd_path = os.path.join(prepdir, "modulemd.yaml")
+        with open(mmd_path, "w") as mmd_f:
+            mmd_f.write(self.mmd)
+        return prepdir
+
+
     def koji_import(self):
+        """This method imports given module into the configured koji instance as
+        a content generator based build
+
+        Raises an exception when error is encountered during import"""
         session = KojiModuleBuilder.get_session(self.config, self.owner)
 
         metadata = self._get_content_generator_metadata()
+        file_dir = self._prepare_file_directory()
+        try:
+            build_info = session.CGImport(metadata, file_dir)
+            log.debug("Content generator import done: %s",
+                      json.dumps(build_info, sort_keys=True, indent=4))
+        except Exception, e:
+            log.error("Content generator import failed: %s", e)
+            raise e
+        finally:
+            shutil.rmtree(file_dir)

tests/__init__.py

@@ -32,6 +32,7 @@ import modulemd
 from module_build_service.utils import get_scm_url_re
 import module_build_service.pdc
 
+base_dir = os.path.dirname(__file__)
 app = module_build_service.app
 conf = init_config(app)
 
@@ -46,7 +47,8 @@ def init_data():
         build_one.stream = '1'
         build_one.version = 2
         build_one.state = 3
-        build_one.modulemd = ''  # Skipping since no tests rely on it
+        with open(os.path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:
+            build_one.modulemd = mmd.read()
         build_one.koji_tag = 'module-nginx-1.2'
         build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?'
                             '#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9')

tests/staged_data/nginx_mmd.yaml

@@ -0,0 +1,31 @@
+# Document type identifier
+document: modulemd
+# Module metadata format version
+version: 1
+data:
+    # Module name, optional
+    # Typically filled in by the buildsystem, using the VCS repository
+    # name as the name of the module.
+    name: nginx
+    # Module update stream, optional
+    # Typically filled in by the buildsystem, using the VCS branch name
+    # as the name of the stream.
+    stream: 1
+    # Module version, integer, optional, cannot be negative
+    # Typically filled in by the buildsystem, using the VCS commit
+    # timestamp.  Module version defines upgrade path for the particular
+    # update stream.
+    version: 2
+    # A short summary describing the module, required
+    summary: An example nginx module
+    # A verbose description of the module, required
+    description: >
+        A module for the tests of module build service
+    # Module and content licenses in the Fedora license identifier
+    # format, required
+    license:
+        # Module license, required
+        # This list covers licenses used for the module metadata, SPEC
+        # files or extra patches
+        module:
+            - MIT

tests/test_content_generator.py

@@ -74,6 +74,7 @@ class TestBuild(unittest.TestCase):
     @patch("platform.machine")
     @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag")
     def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res):
+        """ Test generation of content generator json """
         self.maxDiff = None
         distro.return_value = ("Fedora", "25", "Twenty Five")
         machine.return_value = "i686"
@@ -93,3 +94,10 @@ class TestBuild(unittest.TestCase):
         ret = self.cg._get_content_generator_metadata()
         rpms_in_tag.assert_called_once()
         self.assertEqual(expected_output, ret)
+
+
+    def test_prepare_file_directory(self):
+        """ Test preparation of directory with output files """
+        dir_path = self.cg._prepare_file_directory()
+        with open(path.join(dir_path, "modulemd.yaml")) as mmd:
+            self.assertEqual(len(mmd.read()), 1134)

tests/test_get_generator_json_expected_output.json

@@ -599,8 +599,8 @@
                 }
             ],
             "arch": "noarch",
-            "filesize": 5,
-            "checksum": "ee434023cf89d7dfb21f63d64f0f9d74",
+            "filesize": 1134,
+            "checksum": "bf1615b15f6a0fee485abe94af6b56b6",
             "checksum_type": "md5",
             "type": "modulemd"
         }
@@ -615,7 +615,7 @@
             "typeinfo": {
                 "modulemd": {
                     "module_build_service_id": 1,
-                    "modulemd_str": ""
+                    "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n    # Module name, optional\n    # Typically filled in by the buildsystem, using the VCS repository\n    # name as the name of the module.\n    name: nginx\n    # Module update stream, optional\n    # Typically filled in by the buildsystem, using the VCS branch name\n    # as the name of the stream.\n    stream: 1\n    # Module version, integer, optional, cannot be negative\n    # Typically filled in by the buildsystem, using the VCS commit\n    # timestamp.  Module version defines upgrade path for the particular\n    # update stream.\n    version: 2\n    # A short summary describing the module, required\n    summary: An example nginx module\n    # A verbose description of the module, required\n    description: >\n        A module for the tests of module build service\n    # Module and content licenses in the Fedora license identifier\n    # format, required\n    license:\n        # Module license, required\n        # This list covers licenses used for the module metadata, SPEC\n        # files or extra patches\n        module:\n            - MIT\n"
                 }
             }
         },

tests/test_views/test_views.py

@@ -147,7 +147,8 @@ class TestViews(unittest.TestCase):
         data = json.loads(rv.data)
         self.assertEquals(data['component_builds'], [1, 2])
         self.assertEquals(data['id'], 1)
-        self.assertEquals(data['modulemd'], '')
+        with open(path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:
+            self.assertEquals(data['modulemd'], mmd.read())
         self.assertEquals(data['name'], 'nginx')
         self.assertEquals(data['owner'], 'Moe Szyslak')
         self.assertEquals(data['scmurl'],