Add build logs into the koji content generator

This change adds another output next to the mmd file - a build.log which
will contain builds collected during the module build
This commit is contained in:
Stanislav Ochotnicky
2017-07-03 11:49:40 +02:00
parent db44ce8c3c
commit 747b60f4b4
4 changed files with 725 additions and 4 deletions

View File

@@ -147,6 +147,7 @@ class DevConfiguration(BaseConfiguration):
class TestConfiguration(BaseConfiguration):
BUILD_LOGS_DIR = '/tmp'
LOG_BACKEND = 'console'
LOG_LEVEL = 'debug'
SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(

View File

@@ -36,7 +36,7 @@ import tempfile
import koji
import module_build_service
from module_build_service import log
from module_build_service import log, build_logs
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
logging.basicConfig(level=logging.DEBUG)
@@ -323,6 +323,12 @@ class KojiContentGenerator(object):
mmd_path = os.path.join(prepdir, "modulemd.yaml")
with open(mmd_path, "w") as mmd_f:
mmd_f.write(self.mmd)
log_path = os.path.join(prepdir, "build.log")
try:
shutil.copy(build_logs.path(self.module.id), log_path)
except IOError, e:
log.exception(e)
return prepdir
@@ -333,8 +339,8 @@ class KojiContentGenerator(object):
Raises an exception when error is encountered during import"""
session = KojiModuleBuilder.get_session(self.config, self.owner)
metadata = self._get_content_generator_metadata()
file_dir = self._prepare_file_directory()
metadata = self._get_content_generator_metadata(file_dir)
try:
build_info = session.CGImport(metadata, file_dir)
log.debug("Content generator import done: %s",

View File

@@ -24,6 +24,7 @@ import unittest
import json
import vcr
import os
from os import path
from os.path import dirname
@@ -31,7 +32,7 @@ from os.path import dirname
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
import module_build_service.utils
from module_build_service import models, conf
from module_build_service import models, conf, build_logs
from mock import patch, Mock
@@ -68,6 +69,11 @@ class TestBuild(unittest.TestCase):
del sys.modules['moksha.hub']
import moksha.hub.reactor
self.vcr.__exit__()
try:
path = build_logs.path(self.cg.module.id)
os.remove(path)
except:
pass
@patch("subprocess.Popen")
@patch("pkg_resources.get_distribution")
@@ -76,7 +82,46 @@ class TestBuild(unittest.TestCase):
@patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag")
def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, popen):
""" Test generation of content generator json """
distro.return_value = ("Fedora", "25", "Twenty Five")
self.maxDiff = None
machine.return_value = "i686"
pkg_res.return_value = Mock()
pkg_res.return_value.version = "current-tested-version"
rpm_mock = Mock()
rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \
"rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"
attrs = {'communicate.return_value': (rpm_out, 'error'),
'wait.return_value': 0}
rpm_mock.configure_mock(**attrs)
popen.return_value = rpm_mock
tests_dir = path.abspath(path.dirname(__file__))
rpm_in_tag_path = path.join(tests_dir,
"test_get_generator_json_rpms_in_tag.json")
with open(rpm_in_tag_path) as rpms_in_tag_file:
rpms_in_tag.return_value = json.load(rpms_in_tag_file)
expected_output_path = path.join(tests_dir,
"test_get_generator_json_expected_output_with_log.json")
with open(expected_output_path) as expected_output_file:
expected_output = json.load(expected_output_file)
# create the build.log
build_logs.start(self.cg.module.id)
build_logs.stop(self.cg.module.id)
file_dir = self.cg._prepare_file_directory()
ret = self.cg._get_content_generator_metadata(file_dir)
rpms_in_tag.assert_called_once()
self.assertEqual(expected_output, ret)
@patch("subprocess.Popen")
@patch("pkg_resources.get_distribution")
@patch("platform.linux_distribution")
@patch("platform.machine")
@patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag")
def test_get_generator_json_no_log(self, rpms_in_tag, machine, distro, pkg_res, popen):
""" Test generation of content generator json """
distro.return_value = ("Fedora", "25", "Twenty Five")
machine.return_value = "i686"
pkg_res.return_value = Mock()
@@ -99,7 +144,8 @@ class TestBuild(unittest.TestCase):
"test_get_generator_json_expected_output.json")
with open(expected_output_path) as expected_output_file:
expected_output = json.load(expected_output_file)
ret = self.cg._get_content_generator_metadata()
file_dir = self.cg._prepare_file_directory()
ret = self.cg._get_content_generator_metadata(file_dir)
rpms_in_tag.assert_called_once()
self.assertEqual(expected_output, ret)

View File

@@ -0,0 +1,668 @@
{
"buildroots": [{
"id": 1,
"host": {
"arch": "i686",
"os": "Fedora 25"
},
"content_generator": {
"name": "module-build-service",
"version": "current-tested-version"
},
"tools": [
{
"name": "modulemd",
"version": "current-tested-version"
}
],
"components": [
{
"name": "rpm-name",
"version": "1.0",
"release": "r1",
"epoch": null,
"arch": "x86_64",
"sigmd5": "sigmd5:1",
"signature": "sigpgp:p",
"type": "rpm"
},
{
"name": "rpm-name-2",
"version": "2.0",
"release": "r2",
"epoch": 1,
"arch": "i686",
"sigmd5": "sigmd5:2",
"signature": "sigpgp:p2",
"type": "rpm"
}
],
"container": {
"arch": "i686",
"type": "none"
}
}],
"output": [
{
"filename": "modulemd.yaml",
"buildroot_id": 1,
"components": [
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "armv7hl",
"name": "dhcp-relay",
"release": "5.module_2118aef6",
"sigmd5": "90fa6038158ed88725a1e4d80abf489d"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "armv7hl",
"name": "dhcp-libs",
"release": "5.module_2118aef6",
"sigmd5": "0d5830920551ce9ed6ec3794347f2fce"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "armv7hl",
"name": "dhcp-devel",
"release": "5.module_2118aef6",
"sigmd5": "c705770bb47ef5786c8efa123f5fe797"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "armv7hl",
"name": "dhcp-client",
"release": "5.module_2118aef6",
"sigmd5": "14d454c33f0e69e34af2355297c3c9f0"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "armv7hl",
"name": "dhcp-server",
"release": "5.module_2118aef6",
"sigmd5": "18480b5d37274b933eccaf8600a4c039"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "armv7hl",
"name": "dhcp-compat",
"release": "5.module_2118aef6",
"sigmd5": "1e15de6b5a263bf407eb627f36cc6c0a"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "armv7hl",
"name": "dhcp-debuginfo",
"release": "5.module_2118aef6",
"sigmd5": "2d142f5d634b1595da9f84bfa9998eb8"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "i686",
"name": "dhcp-compat",
"release": "5.module_2118aef6",
"sigmd5": "5bf7e2bb1e457d2d636d39b1f7dd2cd7"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "i686",
"name": "dhcp-relay",
"release": "5.module_2118aef6",
"sigmd5": "4be0015f9290a8c7489f7e8240653730"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "i686",
"name": "dhcp-client",
"release": "5.module_2118aef6",
"sigmd5": "54d581defa0de65df647691cc30b3c8d"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "i686",
"name": "dhcp-server",
"release": "5.module_2118aef6",
"sigmd5": "8b6486cb5ba4dd1355e60d100ff2d269"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "i686",
"name": "dhcp-libs",
"release": "5.module_2118aef6",
"sigmd5": "aaf81ee6c2bce98aa0b9ea7c4a912e08"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "i686",
"name": "dhcp-debuginfo",
"release": "5.module_2118aef6",
"sigmd5": "5ba2750d1411f10be8d4c45bcdae3038"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "i686",
"name": "dhcp-devel",
"release": "5.module_2118aef6",
"sigmd5": "05c0a31a93e89e710d9df04ce943b339"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64",
"name": "dhcp-debuginfo",
"release": "5.module_2118aef6",
"sigmd5": "3dadcbd5643f9228d2c63faa4c7261ac"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64",
"name": "dhcp-client",
"release": "5.module_2118aef6",
"sigmd5": "2d9bf0c0415f5cea9c599b4a263ed6b7"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64",
"name": "dhcp-relay",
"release": "5.module_2118aef6",
"sigmd5": "38d8e6c2cff7c6aab9ce607cd8b881ce"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64",
"name": "dhcp-libs",
"release": "5.module_2118aef6",
"sigmd5": "0ed34d01f24c2e244aee09767d6eec2e"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64",
"name": "dhcp-compat",
"release": "5.module_2118aef6",
"sigmd5": "fb645d96e97c24a4e6650344b38f5eb3"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64",
"name": "dhcp-devel",
"release": "5.module_2118aef6",
"sigmd5": "0a21a17c8230c8c12ff588d4af0892b8"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64",
"name": "dhcp-server",
"release": "5.module_2118aef6",
"sigmd5": "2e5ab6824e0b13a6ca8f89a485cac034"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "aarch64",
"name": "dhcp-relay",
"release": "5.module_2118aef6",
"sigmd5": "1f85196afd24e0664918ca781111e1c5"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "aarch64",
"name": "dhcp-server",
"release": "5.module_2118aef6",
"sigmd5": "e1ef32e7da2c6767b9fb4b623525ab60"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "aarch64",
"name": "dhcp-client",
"release": "5.module_2118aef6",
"sigmd5": "5adae02507dac66db420f8798111a282"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "aarch64",
"name": "dhcp-debuginfo",
"release": "5.module_2118aef6",
"sigmd5": "1310c413efb5e4e6dff196a41b61ab16"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "aarch64",
"name": "dhcp-compat",
"release": "5.module_2118aef6",
"sigmd5": "75be4f00e000f37b2b6ae9361d7a7f65"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "aarch64",
"name": "dhcp-libs",
"release": "5.module_2118aef6",
"sigmd5": "1482a2638fe741086b453cf76645b61b"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "aarch64",
"name": "dhcp-devel",
"release": "5.module_2118aef6",
"sigmd5": "09b21aaf971818463a9d4fa31ade485e"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64le",
"name": "dhcp-server",
"release": "5.module_2118aef6",
"sigmd5": "0c1bed339fe3e71a0252fc69859c76e5"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64le",
"name": "dhcp-client",
"release": "5.module_2118aef6",
"sigmd5": "ae14bf7cac86f5f58ecb18ca2cdeb76e"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64le",
"name": "dhcp-compat",
"release": "5.module_2118aef6",
"sigmd5": "c6dc94a975e8939a73446f9a69da6718"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64le",
"name": "dhcp-relay",
"release": "5.module_2118aef6",
"sigmd5": "4bad7b8404596d0b48be22c9362d7b2c"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64le",
"name": "dhcp-debuginfo",
"release": "5.module_2118aef6",
"sigmd5": "19461ea5d257ffe0dda0dc56cf73ad5d"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64le",
"name": "dhcp-devel",
"release": "5.module_2118aef6",
"sigmd5": "2bcef78ccca5986db7d7fa5e5a9a40b8"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "ppc64le",
"name": "dhcp-libs",
"release": "5.module_2118aef6",
"sigmd5": "b9406e5356db6c36f15af21e4a8ddf7c"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "x86_64",
"name": "dhcp-libs",
"release": "5.module_2118aef6",
"sigmd5": "499ae3c5ca57ef45d14643850da39e52"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "x86_64",
"name": "dhcp-client",
"release": "5.module_2118aef6",
"sigmd5": "b40b18aced3a78a7c670135cb97bed2d"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "x86_64",
"name": "dhcp-debuginfo",
"release": "5.module_2118aef6",
"sigmd5": "68e7c7ac713ba5cd6b49e3912fc74f56"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "x86_64",
"name": "dhcp-devel",
"release": "5.module_2118aef6",
"sigmd5": "96c6c1b0a8bfc782f6dc4451f033cc3f"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "x86_64",
"name": "dhcp-compat",
"release": "5.module_2118aef6",
"sigmd5": "03aaa447e471575b0ff6ba8dbd656ed6"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "x86_64",
"name": "dhcp-server",
"release": "5.module_2118aef6",
"sigmd5": "12fb5fb1c246b3d357239b51034f66f5"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "noarch",
"name": "dhcp-common",
"release": "5.module_2118aef6",
"sigmd5": "d59b5a08ff2ab593978614ecf7f7f0c7"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "x86_64",
"name": "dhcp-relay",
"release": "5.module_2118aef6",
"sigmd5": "846faaa11674763b163c606d0f87a635"
},
{
"epoch": 12,
"type": "rpm",
"version": "4.3.5",
"arch": "src",
"name": "dhcp",
"release": "5.module_2118aef6",
"sigmd5": "45c2f6b6131c68dc4da0833caeefa5d3"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "armv7hl",
"name": "bind99-devel",
"release": "5.P8.module_2118aef6",
"sigmd5": "891e4729ae04ef7d04ead60a884c1c3c"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "armv7hl",
"name": "bind99-debuginfo",
"release": "5.P8.module_2118aef6",
"sigmd5": "1c21d138cf080c00ce2cc5837e7b2b63"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "armv7hl",
"name": "bind99-libs",
"release": "5.P8.module_2118aef6",
"sigmd5": "45f88db7672bd209a9999b1e7a3d6410"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "i686",
"name": "bind99-libs",
"release": "5.P8.module_2118aef6",
"sigmd5": "e2d4a9a7ee3389e0cb354ac9590c573e"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "i686",
"name": "bind99-debuginfo",
"release": "5.P8.module_2118aef6",
"sigmd5": "b2c00bf25e3b943e2f8ff36a904cb6bd"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "i686",
"name": "bind99-devel",
"release": "5.P8.module_2118aef6",
"sigmd5": "d012ca49b66e81ee6d4350b7b27eadf0"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "ppc64",
"name": "bind99-libs",
"release": "5.P8.module_2118aef6",
"sigmd5": "6889d9619b2509d1d6019d4d585e1ace"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "ppc64",
"name": "bind99-devel",
"release": "5.P8.module_2118aef6",
"sigmd5": "758f99d5a885259c05b12422188c3367"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "ppc64",
"name": "bind99-debuginfo",
"release": "5.P8.module_2118aef6",
"sigmd5": "67fb6720b1fdc6a764fc64dc73362e19"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "aarch64",
"name": "bind99-devel",
"release": "5.P8.module_2118aef6",
"sigmd5": "055d1311d31db0279f554630f9c87bc4"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "aarch64",
"name": "bind99-libs",
"release": "5.P8.module_2118aef6",
"sigmd5": "c96f841ab07863b0e5dc40668328515d"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "aarch64",
"name": "bind99-debuginfo",
"release": "5.P8.module_2118aef6",
"sigmd5": "c04123eff71cae1b3ea6b76a44c83d66"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "ppc64le",
"name": "bind99-devel",
"release": "5.P8.module_2118aef6",
"sigmd5": "8db3604b46f68683dd7207e155fe6464"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "ppc64le",
"name": "bind99-libs",
"release": "5.P8.module_2118aef6",
"sigmd5": "629008b3edd9e5ef9a8eedf166351e62"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "ppc64le",
"name": "bind99-debuginfo",
"release": "5.P8.module_2118aef6",
"sigmd5": "b06998cfaf8f9f2b88d3d362472d7c55"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "x86_64",
"name": "bind99-libs",
"release": "5.P8.module_2118aef6",
"sigmd5": "6c9f40725a31320e698267110de19ca9"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "x86_64",
"name": "bind99-debuginfo",
"release": "5.P8.module_2118aef6",
"sigmd5": "ae381be6cccbdad9c15f370550c3e66a"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "x86_64",
"name": "bind99-devel",
"release": "5.P8.module_2118aef6",
"sigmd5": "9845ee96c4a4bfc84eaea2ba46104f5a"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "noarch",
"name": "bind99-license",
"release": "5.P8.module_2118aef6",
"sigmd5": "b60b539e075d68992bfd40346f9f7bd8"
},
{
"epoch": null,
"type": "rpm",
"version": "9.9.9",
"arch": "src",
"name": "bind99",
"release": "5.P8.module_2118aef6",
"sigmd5": "38758ae862424b354b20d9d4d38be97e"
}
],
"arch": "noarch",
"filesize": 1134,
"checksum": "bf1615b15f6a0fee485abe94af6b56b6",
"checksum_type": "md5",
"type": "file",
"extra": {
"typeinfo": {
"module": {}
}
}
},
{
"buildroot_id": 1,
"arch": "noarch",
"type": "log",
"filename": "build.log",
"filesize": 0,
"checksum_type": "md5",
"checksum": "d41d8cd98f00b204e9800998ecf8427e"
}
],
"metadata_version": 0,
"build": {
"version": "1",
"end_time": 1472901932,
"name": "nginx",
"release": "2",
"extra": {
"typeinfo": {
"module": {
"name": "nginx",
"stream": "1",
"version": "2",
"module_build_service_id": 1,
"modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n"
}
}
},
"source": "git://pkgs.domain.local/modules/nginx?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9",
"start_time": 1472901800
}
}