diff --git a/conf/config.py b/conf/config.py index cdb256d8..0c10bbae 100644 --- a/conf/config.py +++ b/conf/config.py @@ -147,6 +147,7 @@ class DevConfiguration(BaseConfiguration): class TestConfiguration(BaseConfiguration): + BUILD_LOGS_DIR = '/tmp' LOG_BACKEND = 'console' LOG_LEVEL = 'debug' SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format( diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 712db3c9..83d8682f 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -36,7 +36,7 @@ import tempfile import koji import module_build_service -from module_build_service import log +from module_build_service import log, build_logs from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder logging.basicConfig(level=logging.DEBUG) @@ -323,6 +323,12 @@ class KojiContentGenerator(object): mmd_path = os.path.join(prepdir, "modulemd.yaml") with open(mmd_path, "w") as mmd_f: mmd_f.write(self.mmd) + + log_path = os.path.join(prepdir, "build.log") + try: + shutil.copy(build_logs.path(self.module.id), log_path) + except IOError, e: + log.exception(e) return prepdir @@ -333,8 +339,8 @@ class KojiContentGenerator(object): Raises an exception when error is encountered during import""" session = KojiModuleBuilder.get_session(self.config, self.owner) - metadata = self._get_content_generator_metadata() file_dir = self._prepare_file_directory() + metadata = self._get_content_generator_metadata(file_dir) try: build_info = session.CGImport(metadata, file_dir) log.debug("Content generator import done: %s", diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index b32d7ca7..79b89606 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -24,6 +24,7 @@ import unittest import json import vcr +import os from os import path from os.path import dirname @@ -31,7 +32,7 @@ from os.path import dirname import module_build_service.messaging import module_build_service.scheduler.handlers.repos import module_build_service.utils -from module_build_service import models, conf +from module_build_service import models, conf, build_logs from mock import patch, Mock @@ -68,6 +69,11 @@ class TestBuild(unittest.TestCase): del sys.modules['moksha.hub'] import moksha.hub.reactor self.vcr.__exit__() + try: + path = build_logs.path(self.cg.module.id) + os.remove(path) + except: + pass @patch("subprocess.Popen") @patch("pkg_resources.get_distribution") @@ -76,7 +82,46 @@ class TestBuild(unittest.TestCase): @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag") def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, popen): """ Test generation of content generator json """ + distro.return_value = ("Fedora", "25", "Twenty Five") self.maxDiff = None + machine.return_value = "i686" + pkg_res.return_value = Mock() + pkg_res.return_value.version = "current-tested-version" + rpm_mock = Mock() + rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \ + "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2" + attrs = {'communicate.return_value': (rpm_out, 'error'), + 'wait.return_value': 0} + rpm_mock.configure_mock(**attrs) + popen.return_value = rpm_mock + + tests_dir = path.abspath(path.dirname(__file__)) + rpm_in_tag_path = path.join(tests_dir, + "test_get_generator_json_rpms_in_tag.json") + with open(rpm_in_tag_path) as rpms_in_tag_file: + rpms_in_tag.return_value = json.load(rpms_in_tag_file) + + expected_output_path = path.join(tests_dir, + "test_get_generator_json_expected_output_with_log.json") + with open(expected_output_path) as expected_output_file: + expected_output = json.load(expected_output_file) + + # create the build.log + build_logs.start(self.cg.module.id) + build_logs.stop(self.cg.module.id) + + file_dir = self.cg._prepare_file_directory() + ret = self.cg._get_content_generator_metadata(file_dir) + rpms_in_tag.assert_called_once() + self.assertEqual(expected_output, ret) + + @patch("subprocess.Popen") + @patch("pkg_resources.get_distribution") + @patch("platform.linux_distribution") + @patch("platform.machine") + @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag") + def test_get_generator_json_no_log(self, rpms_in_tag, machine, distro, pkg_res, popen): + """ Test generation of content generator json """ distro.return_value = ("Fedora", "25", "Twenty Five") machine.return_value = "i686" pkg_res.return_value = Mock() @@ -99,7 +144,8 @@ class TestBuild(unittest.TestCase): "test_get_generator_json_expected_output.json") with open(expected_output_path) as expected_output_file: expected_output = json.load(expected_output_file) - ret = self.cg._get_content_generator_metadata() + file_dir = self.cg._prepare_file_directory() + ret = self.cg._get_content_generator_metadata(file_dir) rpms_in_tag.assert_called_once() self.assertEqual(expected_output, ret) diff --git a/tests/test_get_generator_json_expected_output_with_log.json b/tests/test_get_generator_json_expected_output_with_log.json new file mode 100644 index 00000000..d5a16c73 --- /dev/null +++ b/tests/test_get_generator_json_expected_output_with_log.json @@ -0,0 +1,668 @@ +{ + "buildroots": [{ + "id": 1, + "host": { + "arch": "i686", + "os": "Fedora 25" + }, + "content_generator": { + "name": "module-build-service", + "version": "current-tested-version" + }, + "tools": [ + { + "name": "modulemd", + "version": "current-tested-version" + } + ], + "components": [ + { + "name": "rpm-name", + "version": "1.0", + "release": "r1", + "epoch": null, + "arch": "x86_64", + "sigmd5": "sigmd5:1", + "signature": "sigpgp:p", + "type": "rpm" + }, + { + "name": "rpm-name-2", + "version": "2.0", + "release": "r2", + "epoch": 1, + "arch": "i686", + "sigmd5": "sigmd5:2", + "signature": "sigpgp:p2", + "type": "rpm" + } + ], + "container": { + "arch": "i686", + "type": "none" + } + }], + "output": [ + { + "filename": "modulemd.yaml", + "buildroot_id": 1, + "components": [ + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "armv7hl", + "name": "dhcp-relay", + "release": "5.module_2118aef6", + "sigmd5": "90fa6038158ed88725a1e4d80abf489d" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "armv7hl", + "name": "dhcp-libs", + "release": "5.module_2118aef6", + "sigmd5": "0d5830920551ce9ed6ec3794347f2fce" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "armv7hl", + "name": "dhcp-devel", + "release": "5.module_2118aef6", + "sigmd5": "c705770bb47ef5786c8efa123f5fe797" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "armv7hl", + "name": "dhcp-client", + "release": "5.module_2118aef6", + "sigmd5": "14d454c33f0e69e34af2355297c3c9f0" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "armv7hl", + "name": "dhcp-server", + "release": "5.module_2118aef6", + "sigmd5": "18480b5d37274b933eccaf8600a4c039" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "armv7hl", + "name": "dhcp-compat", + "release": "5.module_2118aef6", + "sigmd5": "1e15de6b5a263bf407eb627f36cc6c0a" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "armv7hl", + "name": "dhcp-debuginfo", + "release": "5.module_2118aef6", + "sigmd5": "2d142f5d634b1595da9f84bfa9998eb8" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "i686", + "name": "dhcp-compat", + "release": "5.module_2118aef6", + "sigmd5": "5bf7e2bb1e457d2d636d39b1f7dd2cd7" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "i686", + "name": "dhcp-relay", + "release": "5.module_2118aef6", + "sigmd5": "4be0015f9290a8c7489f7e8240653730" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "i686", + "name": "dhcp-client", + "release": "5.module_2118aef6", + "sigmd5": "54d581defa0de65df647691cc30b3c8d" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "i686", + "name": "dhcp-server", + "release": "5.module_2118aef6", + "sigmd5": "8b6486cb5ba4dd1355e60d100ff2d269" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "i686", + "name": "dhcp-libs", + "release": "5.module_2118aef6", + "sigmd5": "aaf81ee6c2bce98aa0b9ea7c4a912e08" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "i686", + "name": "dhcp-debuginfo", + "release": "5.module_2118aef6", + "sigmd5": "5ba2750d1411f10be8d4c45bcdae3038" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "i686", + "name": "dhcp-devel", + "release": "5.module_2118aef6", + "sigmd5": "05c0a31a93e89e710d9df04ce943b339" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64", + "name": "dhcp-debuginfo", + "release": "5.module_2118aef6", + "sigmd5": "3dadcbd5643f9228d2c63faa4c7261ac" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64", + "name": "dhcp-client", + "release": "5.module_2118aef6", + "sigmd5": "2d9bf0c0415f5cea9c599b4a263ed6b7" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64", + "name": "dhcp-relay", + "release": "5.module_2118aef6", + "sigmd5": "38d8e6c2cff7c6aab9ce607cd8b881ce" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64", + "name": "dhcp-libs", + "release": "5.module_2118aef6", + "sigmd5": "0ed34d01f24c2e244aee09767d6eec2e" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64", + "name": "dhcp-compat", + "release": "5.module_2118aef6", + "sigmd5": "fb645d96e97c24a4e6650344b38f5eb3" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64", + "name": "dhcp-devel", + "release": "5.module_2118aef6", + "sigmd5": "0a21a17c8230c8c12ff588d4af0892b8" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64", + "name": "dhcp-server", + "release": "5.module_2118aef6", + "sigmd5": "2e5ab6824e0b13a6ca8f89a485cac034" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "aarch64", + "name": "dhcp-relay", + "release": "5.module_2118aef6", + "sigmd5": "1f85196afd24e0664918ca781111e1c5" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "aarch64", + "name": "dhcp-server", + "release": "5.module_2118aef6", + "sigmd5": "e1ef32e7da2c6767b9fb4b623525ab60" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "aarch64", + "name": "dhcp-client", + "release": "5.module_2118aef6", + "sigmd5": "5adae02507dac66db420f8798111a282" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "aarch64", + "name": "dhcp-debuginfo", + "release": "5.module_2118aef6", + "sigmd5": "1310c413efb5e4e6dff196a41b61ab16" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "aarch64", + "name": "dhcp-compat", + "release": "5.module_2118aef6", + "sigmd5": "75be4f00e000f37b2b6ae9361d7a7f65" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "aarch64", + "name": "dhcp-libs", + "release": "5.module_2118aef6", + "sigmd5": "1482a2638fe741086b453cf76645b61b" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "aarch64", + "name": "dhcp-devel", + "release": "5.module_2118aef6", + "sigmd5": "09b21aaf971818463a9d4fa31ade485e" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64le", + "name": "dhcp-server", + "release": "5.module_2118aef6", + "sigmd5": "0c1bed339fe3e71a0252fc69859c76e5" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64le", + "name": "dhcp-client", + "release": "5.module_2118aef6", + "sigmd5": "ae14bf7cac86f5f58ecb18ca2cdeb76e" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64le", + "name": "dhcp-compat", + "release": "5.module_2118aef6", + "sigmd5": "c6dc94a975e8939a73446f9a69da6718" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64le", + "name": "dhcp-relay", + "release": "5.module_2118aef6", + "sigmd5": "4bad7b8404596d0b48be22c9362d7b2c" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64le", + "name": "dhcp-debuginfo", + "release": "5.module_2118aef6", + "sigmd5": "19461ea5d257ffe0dda0dc56cf73ad5d" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64le", + "name": "dhcp-devel", + "release": "5.module_2118aef6", + "sigmd5": "2bcef78ccca5986db7d7fa5e5a9a40b8" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "ppc64le", + "name": "dhcp-libs", + "release": "5.module_2118aef6", + "sigmd5": "b9406e5356db6c36f15af21e4a8ddf7c" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "x86_64", + "name": "dhcp-libs", + "release": "5.module_2118aef6", + "sigmd5": "499ae3c5ca57ef45d14643850da39e52" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "x86_64", + "name": "dhcp-client", + "release": "5.module_2118aef6", + "sigmd5": "b40b18aced3a78a7c670135cb97bed2d" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "x86_64", + "name": "dhcp-debuginfo", + "release": "5.module_2118aef6", + "sigmd5": "68e7c7ac713ba5cd6b49e3912fc74f56" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "x86_64", + "name": "dhcp-devel", + "release": "5.module_2118aef6", + "sigmd5": "96c6c1b0a8bfc782f6dc4451f033cc3f" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "x86_64", + "name": "dhcp-compat", + "release": "5.module_2118aef6", + "sigmd5": "03aaa447e471575b0ff6ba8dbd656ed6" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "x86_64", + "name": "dhcp-server", + "release": "5.module_2118aef6", + "sigmd5": "12fb5fb1c246b3d357239b51034f66f5" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "noarch", + "name": "dhcp-common", + "release": "5.module_2118aef6", + "sigmd5": "d59b5a08ff2ab593978614ecf7f7f0c7" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "x86_64", + "name": "dhcp-relay", + "release": "5.module_2118aef6", + "sigmd5": "846faaa11674763b163c606d0f87a635" + }, + { + "epoch": 12, + "type": "rpm", + "version": "4.3.5", + "arch": "src", + "name": "dhcp", + "release": "5.module_2118aef6", + "sigmd5": "45c2f6b6131c68dc4da0833caeefa5d3" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "armv7hl", + "name": "bind99-devel", + "release": "5.P8.module_2118aef6", + "sigmd5": "891e4729ae04ef7d04ead60a884c1c3c" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "armv7hl", + "name": "bind99-debuginfo", + "release": "5.P8.module_2118aef6", + "sigmd5": "1c21d138cf080c00ce2cc5837e7b2b63" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "armv7hl", + "name": "bind99-libs", + "release": "5.P8.module_2118aef6", + "sigmd5": "45f88db7672bd209a9999b1e7a3d6410" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "i686", + "name": "bind99-libs", + "release": "5.P8.module_2118aef6", + "sigmd5": "e2d4a9a7ee3389e0cb354ac9590c573e" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "i686", + "name": "bind99-debuginfo", + "release": "5.P8.module_2118aef6", + "sigmd5": "b2c00bf25e3b943e2f8ff36a904cb6bd" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "i686", + "name": "bind99-devel", + "release": "5.P8.module_2118aef6", + "sigmd5": "d012ca49b66e81ee6d4350b7b27eadf0" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "ppc64", + "name": "bind99-libs", + "release": "5.P8.module_2118aef6", + "sigmd5": "6889d9619b2509d1d6019d4d585e1ace" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "ppc64", + "name": "bind99-devel", + "release": "5.P8.module_2118aef6", + "sigmd5": "758f99d5a885259c05b12422188c3367" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "ppc64", + "name": "bind99-debuginfo", + "release": "5.P8.module_2118aef6", + "sigmd5": "67fb6720b1fdc6a764fc64dc73362e19" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "aarch64", + "name": "bind99-devel", + "release": "5.P8.module_2118aef6", + "sigmd5": "055d1311d31db0279f554630f9c87bc4" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "aarch64", + "name": "bind99-libs", + "release": "5.P8.module_2118aef6", + "sigmd5": "c96f841ab07863b0e5dc40668328515d" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "aarch64", + "name": "bind99-debuginfo", + "release": "5.P8.module_2118aef6", + "sigmd5": "c04123eff71cae1b3ea6b76a44c83d66" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "ppc64le", + "name": "bind99-devel", + "release": "5.P8.module_2118aef6", + "sigmd5": "8db3604b46f68683dd7207e155fe6464" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "ppc64le", + "name": "bind99-libs", + "release": "5.P8.module_2118aef6", + "sigmd5": "629008b3edd9e5ef9a8eedf166351e62" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "ppc64le", + "name": "bind99-debuginfo", + "release": "5.P8.module_2118aef6", + "sigmd5": "b06998cfaf8f9f2b88d3d362472d7c55" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "x86_64", + "name": "bind99-libs", + "release": "5.P8.module_2118aef6", + "sigmd5": "6c9f40725a31320e698267110de19ca9" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "x86_64", + "name": "bind99-debuginfo", + "release": "5.P8.module_2118aef6", + "sigmd5": "ae381be6cccbdad9c15f370550c3e66a" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "x86_64", + "name": "bind99-devel", + "release": "5.P8.module_2118aef6", + "sigmd5": "9845ee96c4a4bfc84eaea2ba46104f5a" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "noarch", + "name": "bind99-license", + "release": "5.P8.module_2118aef6", + "sigmd5": "b60b539e075d68992bfd40346f9f7bd8" + }, + { + "epoch": null, + "type": "rpm", + "version": "9.9.9", + "arch": "src", + "name": "bind99", + "release": "5.P8.module_2118aef6", + "sigmd5": "38758ae862424b354b20d9d4d38be97e" + } + ], + "arch": "noarch", + "filesize": 1134, + "checksum": "bf1615b15f6a0fee485abe94af6b56b6", + "checksum_type": "md5", + "type": "file", + "extra": { + "typeinfo": { + "module": {} + } + } + }, + { + "buildroot_id": 1, + "arch": "noarch", + "type": "log", + "filename": "build.log", + "filesize": 0, + "checksum_type": "md5", + "checksum": "d41d8cd98f00b204e9800998ecf8427e" + } + ], + "metadata_version": 0, + "build": { + "version": "1", + "end_time": 1472901932, + "name": "nginx", + "release": "2", + "extra": { + "typeinfo": { + "module": { + "name": "nginx", + "stream": "1", + "version": "2", + "module_build_service_id": 1, + "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n" + } + } + }, + "source": "git://pkgs.domain.local/modules/nginx?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9", + "start_time": 1472901800 + } +}