modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 03:38:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix various bugs in 'mbs-build local'

Browse Source
This commit is contained in:
Jan Kaluza
2017-06-07 12:31:49 +02:00
parent a07b44411c
commit 4b16cf9a64
6 changed files with 69 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
conf/config.py
View File

@@ -34,7 +34,7 @@ class BaseConfiguration(object):
KOJI_TAG_PREFIXES = ['module']
KOJI_ENABLE_CONTENT_GENERATOR = True
COPR_CONFIG = '/etc/module-build-service/copr.conf'
PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1'
PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1'
PDC_INSECURE = True
PDC_DEVELOP = True
SCMURLS = ["git://pkgs.fedoraproject.org/modules/"]
@@ -153,7 +153,7 @@ class TestConfiguration(BaseConfiguration):
path.join(dbdir, 'tests', 'test_module_build_service.db'))
DEBUG = True
MESSAGING = 'in_memory'
PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1'
PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1'
# Global network-related values, in seconds
NET_TIMEOUT = 3
@@ -170,3 +170,7 @@ class TestConfiguration(BaseConfiguration):
class ProdConfiguration(BaseConfiguration):
pass
class LocalBuildConfiguration(BaseConfiguration):
LOG_LEVEL = 'debug'
MESSAGING = 'in_memory'

50
module_build_service/builder/MockModuleBuilder.py
View File

@@ -164,8 +164,15 @@ mdpolicy=group:primary
# Generate the mmd the same way as pungi does.
m1 = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one()
m1_mmd = m1.mmd()
for rpm in os.listdir(self.resultsdir):
if not rpm.endswith(".rpm"):
continue
rpm = rpm[:-len(".rpm")]
m1_mmd.artifacts.add_rpm(str(rpm))
mmd_path = os.path.join(path, "modules.yaml")
modulemd.dump_all(mmd_path, [ m1.mmd() ])
modulemd.dump_all(mmd_path, [ m1_mmd ])
# Generate repo and inject modules.yaml there.
execute_cmd(['/usr/bin/createrepo_c', path])
@@ -282,7 +289,10 @@ mdpolicy=group:primary
self._write_mock_config()
def _send_build_change(self, state, source, build_id):
nvr = kobo.rpmlib.parse_nvr(source)
try:
nvr = kobo.rpmlib.parse_nvr(source)
except ValueError:
nvr = {"name": source, "release": "unknown", "version": "unknown"}
# build_id=1 and task_id=1 are OK here, because we are building just
# one RPM at the time.
@@ -319,13 +329,14 @@ mdpolicy=group:primary
mock_stderr_log = open(os.path.join(self.resultsdir,
artifact_name + "-mock-stderr.log"), "w")
srpm = artifact_name
resultsdir = builder.resultsdir
try:
# Initialize mock.
execute_cmd(["mock", "-v", "-r", mock_config, "--init"],
stdout=mock_stdout_log, stderr=mock_stderr_log)
# Start the build and store results to resultsdir
resultsdir = builder.resultsdir
builder.build(mock_stdout_log, mock_stderr_log)
srpm = find_srpm(resultsdir)
@@ -347,7 +358,7 @@ mdpolicy=group:primary
# by MBS after the build_srpm() method returns and scope gets
# back to scheduler.main.main() method.
state = koji.BUILD_STATES['FAILED']
self._send_build_change(state, source,
self._send_build_change(state, srpm,
build_id)
with open(os.path.join(resultsdir, "status.log"), 'w') as f:
f.write("failed\n")
@@ -445,15 +456,38 @@ class SCMBuilder(BaseBuilder):
branch = source.split("?#")[1]
distgit_cmds = self._get_distgit_commands(source)
distgit_get = distgit_cmds[0].format(artifact_name)
# mock-scm cannot checkout particular commit hash, but only branch.
# We therefore create distgit-clone-wrapper script which clones
# the repository and checkouts particular commit hash.
# See https://bugzilla.redhat.com/show_bug.cgi?id=1459437 for
# more info. Once mock-scm supports this feature, we can remove
# this code.
wrapper_path = os.path.join(os.path.dirname(config), "distgit-clone-wrapper")
with open(wrapper_path, "w") as fd:
fd.writelines([
"#!/bin/sh -eu\n",
"%s\n" % distgit_get,
"git -C $1 checkout $2\n",
])
self._make_executable(wrapper_path)
f.writelines([
"config_opts['scm'] = True\n",
"config_opts['scm_opts']['method'] = 'distgit'\n",
"config_opts['scm_opts']['branch'] = '{}'\n".format(branch),
"config_opts['scm_opts']['package'] = '{}'\n".format(artifact_name),
"config_opts['scm_opts']['distgit_get'] = '{}'\n".format(distgit_get),
"config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format(distgit_cmds[1]),
"config_opts['scm_opts']['package'] = '{}'\n".format(
artifact_name),
"config_opts['scm_opts']['distgit_get'] = '{} {} {}'\n".format(
wrapper_path, artifact_name, branch),
"config_opts['scm_opts']['distgit_src_get'] = '{}'\n".format(
distgit_cmds[1]),
])
def _make_executable(self, path):
mode = os.stat(path).st_mode
mode |= (mode & 0o444) >> 2 # copy R bits to X
os.chmod(path, mode)
def _get_distgit_commands(self, source):
for host, cmds in conf.distgits.items():
if source.startswith(host):

10
module_build_service/config.py
View File

@@ -51,6 +51,12 @@ def init_config(app):
if flask_app_env and any([var.startswith('mod_wsgi.')
for var in app.request.environ]):
config_section = 'ProdConfiguration'
# Load LocalBuildConfiguration section in case we are building modules
# locally.
if "build_module_locally" in sys.argv:
config_section = "LocalBuildConfiguration"
# try getting config_file from os.environ
if 'MBS_CONFIG_FILE' in os.environ:
config_file = os.environ['MBS_CONFIG_FILE']
@@ -294,7 +300,9 @@ class Config(object):
'distgits': {
'type': dict,
'default': {
'git://pkgs.fedoraproject.org': ('fedpkg clone --anonymous {}', 'fedpkg sources'),
'git://pkgs.fedoraproject.org':
('fedpkg clone --anonymous $1',
'fedpkg --release module sources'),
},
'desc': 'Mapping between dist-git and command to '},
'mock_config': {

3
module_build_service/manage.py
View File

@@ -152,10 +152,9 @@ def build_module_locally(url, branch):
submit_module_build_from_scm(username, url, branch, allow_local_url=True)
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
initial_messages = [MBSModule("local module build", 1, 1)]
# Run the consumer until stop_condition returns True
module_build_service.scheduler.main(initial_messages, stop)
module_build_service.scheduler.main([], stop)
@manager.command

2
module_build_service/pdc.py
View File

@@ -173,7 +173,7 @@ def get_module(session, module_info, strict=False):
if module_info.get('active'):
query['active'] = module_info['active']
retval = session['unreleasedvariants'](page_size=-1, **query) # ordering=variant_release...
retval = session['unreleasedvariants/'](page_size=-1, **query) # ordering=variant_release...
# Error handling
if not retval:

13
module_build_service/utils.py
View File

@@ -107,12 +107,14 @@ def start_build_component(builder, c):
Submits single component build to builder. Called in thread
by QueueBasedThreadPool in continue_batch_build.
"""
import koji
try:
c.task_id, c.state, c.state_reason, c.nvr = builder.build(
artifact_name=c.package, source=c.scmurl)
except Exception as e:
c.state = koji.BUILD_STATES['FAILED']
c.state_reason = "Failed to build artifact %s: %s" % (c.package, str(e))
log.exception(e)
return
if not c.task_id and c.state == koji.BUILD_STATES['BUILDING']:
@@ -180,6 +182,11 @@ def continue_batch_build(config, module, session, builder, components=None):
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {executor.submit(start_build_component, builder, c): c for c in components_to_build}
concurrent.futures.wait(futures)
# In case there has been an excepion generated directly in the
# start_build_component, the future.result() will re-raise it in the
# main thread so it is not lost.
for future in futures:
future.result()
# If all components in this batch are already done, it can mean that they
# have been built in the past and have been skipped in this module build.
@@ -946,15 +953,15 @@ def get_reusable_component(session, module, component_name):
# Perform a sanity check to make sure that the buildrequires are the same
# as the buildrequires in xmd for the passed in mmd
if mmd.buildrequires.keys() != mmd.xmd['mbs']['buildrequires'].keys():
if set(mmd.buildrequires.keys()) != set(mmd.xmd['mbs']['buildrequires'].keys()):
log.error(
'The submitted module "{0}" has different keys in mmd.buildrequires'
' than in mmd.xmd[\'mbs\'][\'buildrequires\']'.format(mmd.name))
return None
# Perform a sanity check to make sure that the buildrequires are the same
# as the buildrequires in xmd for the mmd of the previous module build
if old_mmd.buildrequires.keys() != \
old_mmd.xmd['mbs']['buildrequires'].keys():
if set(old_mmd.buildrequires.keys()) != \
set(old_mmd.xmd['mbs']['buildrequires'].keys()):
log.error(
'Version "{0}" of the module "{1}" has different keys in '
'mmd.buildrequires than in mmd.xmd[\'mbs\'][\'buildrequires\']'