From f5e3c81b0c856aab678cfa590b6c9bb14e24d66c Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Tue, 20 Nov 2018 12:47:44 +0100 Subject: [PATCH] Import original source modulemd file as stored in dist-git to CG build. There is a need to ship the unchanged source file which was used to build a module build from legal reasons. The KojiContentGenerator is extended in this commit to fetch it from SCM URL using the `scm` module and later attach it as `modulemd.src.txt` to Koji CG build. --- .../builder/KojiContentGenerator.py | 74 ++++++++++++++++--- tests/test_content_generator.py | 14 ++++ 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index a4305785..6f490aba 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -42,6 +42,7 @@ import koji import pungi.arch from module_build_service import conf, log, build_logs, Modulemd, glib +from module_build_service.scm import SCM logging.basicConfig(level=logging.DEBUG) @@ -366,16 +367,27 @@ class KojiContentGenerator(object): # Read the modulemd file to get the filesize/checksum and also # parse it to get the Modulemd instance. mmd_path = os.path.join(output_path, mmd_filename) - with open(mmd_path) as mmd_f: - data = mmd_f.read() - mmd = Modulemd.Module().new_from_string(data) - ret['filename'] = mmd_filename - ret['filesize'] = len(data) - ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest() + try: + with open(mmd_path) as mmd_f: + data = mmd_f.read() + mmd = Modulemd.Module().new_from_string(data) + ret['filename'] = mmd_filename + ret['filesize'] = len(data) + ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest() + except IOError: + if arch == "src": + # This might happen in case the Module is submitted directly + # using the yaml without SCM URL. This should never happen + # when building production-ready modules using Koji, but in + # theory it is possible. + log.warn("No modulemd.src.txt found.") + return + else: + raise components = [] - if arch == "noarch": - # For generic noarch modulemd, include all the RPMs. + if arch in ["noarch", "src"]: + # For generic noarch/src modulemd, include all the RPMs. for rpm in self.rpms: components.append( self._koji_rpm_to_component_record(rpm)) @@ -397,8 +409,10 @@ class KojiContentGenerator(object): def _get_output(self, output_path): ret = [] - for arch in self.arches + ["noarch"]: - ret.append(self._get_arch_mmd_output(output_path, arch)) + for arch in self.arches + ["noarch", "src"]: + mmd_dict = self._get_arch_mmd_output(output_path, arch) + if mmd_dict: + ret.append(mmd_dict) try: log_path = os.path.join(output_path, "build.log") @@ -613,6 +627,43 @@ class KojiContentGenerator(object): return unicode(mmd.dumps()) + def _download_source_modulemd(self, mmd, output_path): + """ + Fetches the original source modulemd file from SCM URL stored in the + XMD section of `mmd` and stores it to filename referenced by `output_path`. + + This method does nothing if SCM URL is not set in the `mmd`. + + :param Modulemd mmd: Modulemd instance. + :param str output_path: Full path to file into which the original modulemd + file will be stored. + """ + xmd = glib.from_variant_dict(mmd.get_xmd()) + commit = xmd.get("mbs", {}).get("commit") + scmurl = xmd.get("mbs", {}).get("scmurl") + if not commit or not scmurl: + log.warn("%r: xmd['mbs'] does not contain 'commit' or 'scmurl'.", self.module) + return + + td = None + try: + log.info("Fetching %s (%s) to get the source modulemd.yaml", scmurl, commit) + td = tempfile.mkdtemp() + scm = SCM(scmurl) + scm.commit = commit + scm.checkout(td) + fn = scm.get_module_yaml() + log.info("Writing source modulemd.yaml to %r" % output_path) + shutil.copy(fn, output_path) + finally: + try: + if td is not None: + shutil.rmtree(td) + except Exception as e: + log.warn( + "Failed to remove temporary directory {!r}: {}".format( + td, str(e))) + def _prepare_file_directory(self): """ Creates a temporary directory that will contain all the files mentioned in the outputs section @@ -625,6 +676,9 @@ class KojiContentGenerator(object): with open(mmd_path, "w") as mmd_f: mmd_f.write(self.mmd) + mmd_path = os.path.join(prepdir, "modulemd.src.txt") + self._download_source_modulemd(self.module.mmd(), mmd_path) + for arch in self.arches: mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch) log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path)) diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index 7b23efe7..867c7cdf 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -35,6 +35,7 @@ from mock import patch, Mock, MagicMock, call, mock_open import kobo.rpmlib from tests import init_data +from tests.test_views.test_views import FakeSCM from module_build_service.builder.KojiContentGenerator import KojiContentGenerator @@ -630,3 +631,16 @@ class TestBuild: assert pkg.get_cache() is None assert "mbs" not in mmd.get_xmd().keys() + + @patch('module_build_service.builder.KojiContentGenerator.SCM') + def test_prepare_file_directory_modulemd_src(self, mocked_scm): + FakeSCM(mocked_scm, 'testmodule', 'testmodule_init.yaml', + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + mmd = self.cg.module.mmd() + mmd.set_xmd(glib.dict_values({"mbs": { + "commit": "foo", + "scmurl": "git://localhost/modules/foo.git#master"}})) + self.cg.module.modulemd = mmd.dumps() + file_dir = self.cg._prepare_file_directory() + with open(path.join(file_dir, "modulemd.src.txt")) as mmd: + assert len(mmd.read()) == 1337