modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-14 15:19:45 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Port to libmodulemd and support v2 modulemd without module stream expansion

Browse Source
This commit is contained in:
mprahl
2018-02-27 13:29:00 -05:00
parent 329a7acd76
commit f0852d9009
26 changed files with 591 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
module_build_service/utils.py
View File

@@ -33,12 +33,13 @@ import kobo.rpmlib
import inspect
import hashlib
from functools import wraps
import modulemd
import yaml
from flask import request, url_for, Response
from datetime import datetime
from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from module_build_service import log, models
from module_build_service.errors import (ValidationError, UnprocessableEntity,
@@ -48,6 +49,7 @@ from module_build_service.errors import (Forbidden, Conflict)
import module_build_service.messaging
from multiprocessing.dummy import Pool as ThreadPool
import module_build_service.resolver
from module_build_service import glib
import concurrent.futures
@@ -558,7 +560,6 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False):
# and fails to import them because of dep-chain.
import module_build_service.scm
yaml = ""
td = None
scm = None
try:
@@ -571,9 +572,7 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False):
scm.checkout(td)
scm.verify()
cofn = scm.get_module_yaml()
with open(cofn, "r") as mmdfile:
yaml = mmdfile.read()
mmd = load_mmd(cofn, is_file=True)
finally:
try:
if td is not None:
@@ -583,45 +582,65 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False):
"Failed to remove temporary directory {!r}: {}".format(
td, str(e)))
mmd = load_mmd(yaml)
# If the name was set in the modulemd, make sure it matches what the scmurl
# says it should be
if mmd.name and mmd.name != scm.name:
if mmd.get_name() and mmd.get_name() != scm.name:
raise ValidationError('The name "{0}" that is stored in the modulemd '
'is not valid'.format(mmd.name))
'is not valid'.format(mmd.get_name()))
else:
mmd.name = scm.name
mmd.set_name(scm.name)
# If the stream was set in the modulemd, make sure it matches what the repo
# branch is
if mmd.stream and mmd.stream != scm.branch:
if mmd.get_stream() and mmd.get_stream() != scm.branch:
raise ValidationError('The stream "{0}" that is stored in the modulemd '
'does not match the branch "{1}"'.format(
mmd.stream, scm.branch))
mmd.get_stream(), scm.branch))
else:
mmd.stream = str(scm.branch)
mmd.set_stream(str(scm.branch))
# If the version is in the modulemd, throw an exception since the version
# is generated by pdc-updater
if mmd.version:
if mmd.get_version():
raise ValidationError('The version "{0}" is already defined in the '
'modulemd but it shouldn\'t be since the version '
'is generated based on the commit time'.format(
mmd.version))
mmd.get_version()))
else:
mmd.version = int(scm.version)
mmd.set_version(int(scm.version))
return mmd, scm
def load_mmd(yaml):
mmd = modulemd.ModuleMetadata()
def load_mmd(yaml, is_file=False):
try:
mmd.loads(yaml)
except Exception as e:
log.error('Invalid modulemd: %s' % str(e))
raise UnprocessableEntity('Invalid modulemd: %s' % str(e))
if is_file:
mmd = Modulemd.Module().new_from_file(yaml)
else:
mmd = Modulemd.Module().new_from_string(yaml)
# If the modulemd was v1, it will be upgraded to v2
mmd.upgrade()
except Exception:
error = 'The following invalid modulemd was encountered: {0}'.format(yaml)
log.error(error)
raise UnprocessableEntity(error)
# MBS doesn't support module stream expansion yet but supports the v2 modulemd format,
# so if module stream expansion syntax is used, fail the submission
# TODO: Once module stream expansion is supported, the get_dependencies() function should
# be squashed to a single list on resulting modulemds
error_msg = 'Module stream expansion is not yet supported in MBS'
deps_list = mmd.get_dependencies()
if len(deps_list) > 1:
raise UnprocessableEntity(error_msg)
elif len(deps_list) == 1:
for dep_type in ['requires', 'buildrequires']:
deps = getattr(deps_list[0], 'get_{0}'.format(dep_type))()
for streams in deps.values():
if len(streams.get()) != 1:
raise UnprocessableEntity(error_msg)
elif streams.get()[0].startswith('-'):
raise UnprocessableEntity(error)
return mmd
@@ -632,15 +651,14 @@ def _scm_get_latest(pkg):
# to the specific commit available at the time of
# submission (now).
pkgref = module_build_service.scm.SCM(
pkg.repository).get_latest(pkg.ref)
pkg.get_repository()).get_latest(pkg.get_ref())
except Exception as e:
log.exception(e)
return {
'error': "Failed to get the latest commit for %s#%s" % (pkg.repository, pkg.ref)
}
return {'error': "Failed to get the latest commit for %s#%s" % (
pkg.get_repository(), pkg.get_ref())}
return {
'pkg_name': pkg.name,
'pkg_name': pkg.get_name(),
'pkg_ref': pkgref,
'error': None
}
@@ -711,19 +729,15 @@ def load_local_builds(local_build_nsvs, session=None):
# Load the modulemd metadata.
path = os.path.join(conf.mock_resultsdir, found_build[3], 'results')
mmd_path = os.path.join(path, 'modules.yaml')
with open(mmd_path, 'r') as f:
mmd_data = yaml.safe_load(f)
mmd = modulemd.ModuleMetadata()
mmd.loadd(mmd_data)
mmd = load_mmd(os.path.join(path, 'modules.yaml'), is_file=True)
# Create ModuleBuild in database.
module = models.ModuleBuild.create(
session,
conf,
name=mmd.name,
stream=mmd.stream,
version=str(mmd.version),
name=mmd.get_name(),
stream=mmd.get_stream(),
version=str(mmd.get_version()),
modulemd=mmd.dumps(),
scmurl="",
username="mbs",
@@ -755,7 +769,8 @@ def format_mmd(mmd, scmurl, session=None):
if not session:
session = db.session
mmd.xmd['mbs'] = {'scmurl': scmurl, 'commit': None}
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs'] = {'scmurl': scmurl or '', 'commit': ''}
local_modules = models.ModuleBuild.local_modules(session)
local_modules = {m.name + "-" + m.stream: m for m in local_modules}
@@ -771,62 +786,56 @@ def format_mmd(mmd, scmurl, session=None):
else:
full_scm_hash = scm.get_full_commit_hash()
mmd.xmd['mbs']['commit'] = full_scm_hash
xmd['mbs']['commit'] = full_scm_hash
# If a commit hash wasn't provided then just get the latest from master
else:
mmd.xmd['mbs']['commit'] = scm.get_latest()
xmd['mbs']['commit'] = scm.get_latest()
resolver = module_build_service.resolver.GenericResolver.create(conf)
# Resolve Build-requires.
if mmd.buildrequires:
mmd.xmd['mbs']['buildrequires'] = resolver.resolve_requires(
mmd.buildrequires)
else:
mmd.xmd['mbs']['buildrequires'] = {}
# Resolve buildrequires and requires
# Reformat the input for resolve_requires to match the old modulemd format
dep_obj = mmd.get_dependencies()[0]
br_dict = {br: br_list.get()[0] for br, br_list in dep_obj.get_buildrequires().items()}
req_dict = {req: req_list.get()[0] for req, req_list in dep_obj.get_requires().items()}
xmd['mbs']['buildrequires'] = resolver.resolve_requires(br_dict)
xmd['mbs']['requires'] = resolver.resolve_requires(req_dict)
# Resolve Requires.
if mmd.requires:
mmd.xmd['mbs']['requires'] = resolver.resolve_requires(mmd.requires)
else:
mmd.xmd['mbs']['requires'] = {}
if mmd.components:
if 'rpms' not in mmd.xmd['mbs']:
mmd.xmd['mbs']['rpms'] = {}
if mmd.get_rpm_components() or mmd.get_module_components():
if 'rpms' not in xmd['mbs']:
xmd['mbs']['rpms'] = {}
# Add missing data in RPM components
for pkgname, pkg in mmd.components.rpms.items():
if pkg.repository and not conf.rpms_allow_repository:
for pkgname, pkg in mmd.get_rpm_components().items():
if pkg.get_repository() and not conf.rpms_allow_repository:
raise Forbidden(
"Custom component repositories aren't allowed. "
"%r bears repository %r" % (pkgname, pkg.repository))
if pkg.cache and not conf.rpms_allow_cache:
"%r bears repository %r" % (pkgname, pkg.get_repository()))
if pkg.get_cache() and not conf.rpms_allow_cache:
raise Forbidden(
"Custom component caches aren't allowed. "
"%r bears cache %r" % (pkgname, pkg.cache))
if not pkg.repository:
pkg.repository = conf.rpms_default_repository + pkgname
if not pkg.cache:
pkg.cache = conf.rpms_default_cache + pkgname
if not pkg.ref:
pkg.ref = 'master'
if not pkg.get_repository():
pkg.set_repository(conf.rpms_default_repository + pkgname)
if not pkg.get_cache():
pkg.set_cache(conf.rpms_default_cache + pkgname)
if not pkg.get_ref():
pkg.set_ref('master')
# Add missing data in included modules components
for modname, mod in mmd.components.modules.items():
if mod.repository and not conf.modules_allow_repository:
for modname, mod in mmd.get_module_components().items():
if mod.get_repository() and not conf.modules_allow_repository:
raise Forbidden(
"Custom module repositories aren't allowed. "
"%r bears repository %r" % (modname, mod.repository))
if not mod.repository:
mod.repository = conf.modules_default_repository + modname
if not mod.ref:
mod.ref = 'master'
"%r bears repository %r" % (modname, mod.get_repository()))
if not mod.get_repository():
mod.set_repository(conf.modules_default_repository + modname)
if not mod.get_ref():
mod.set_ref('master')
# Check that SCM URL is valid and replace potential branches in
# pkg.ref by real SCM hash and store the result to our private xmd
# place in modulemd.
# Check that SCM URL is valid and replace potential branches in pkg refs
# by real SCM hash and store the result to our private xmd place in modulemd.
pool = ThreadPool(20)
pkg_dicts = pool.map(_scm_get_latest, mmd.components.rpms.values())
pkg_dicts = pool.map(_scm_get_latest, mmd.get_rpm_components().values())
err_msg = ""
for pkg_dict in pkg_dicts:
if pkg_dict["error"]:
@@ -834,17 +843,20 @@ def format_mmd(mmd, scmurl, session=None):
else:
pkg_name = pkg_dict["pkg_name"]
pkg_ref = pkg_dict["pkg_ref"]
mmd.xmd['mbs']['rpms'][pkg_name] = {'ref': pkg_ref}
xmd['mbs']['rpms'][pkg_name] = {'ref': pkg_ref}
if err_msg:
raise UnprocessableEntity(err_msg)
# Set the modified xmd back to the modulemd
mmd.set_xmd(glib.dict_values(xmd))
def validate_mmd(mmd):
for modname, mod in mmd.components.modules.items():
if mod.repository and not conf.modules_allow_repository:
for modname, mod in mmd.get_module_components().items():
if mod.get_repository() and not conf.modules_allow_repository:
raise Forbidden(
"Custom module repositories aren't allowed. "
"%r bears repository %r" % (modname, mod.repository))
"%r bears repository %r" % (modname, mod.get_repository()))
def merge_included_mmd(mmd, included_mmd):
@@ -852,11 +864,15 @@ def merge_included_mmd(mmd, included_mmd):
Merges two modulemds. This merges only metadata which are needed in
the `main` when it includes another module defined by `included_mmd`
"""
if 'rpms' in included_mmd.xmd['mbs']:
if 'rpms' not in mmd.xmd['mbs']:
mmd.xmd['mbs']['rpms'] = included_mmd.xmd['mbs']['rpms']
included_xmd = glib.from_variant_dict(included_mmd.get_xmd())
if 'rpms' in included_xmd['mbs']:
xmd = glib.from_variant_dict(mmd.get_xmd())
if 'rpms' not in xmd['mbs']:
xmd['mbs']['rpms'] = included_xmd['mbs']['rpms']
else:
mmd.xmd['mbs']['rpms'].update(included_mmd.xmd['mbs']['rpms'])
xmd['mbs']['rpms'].update(included_xmd['mbs']['rpms'])
# Set the modified xmd back to the modulemd
mmd.set_xmd(glib.dict_values(xmd))
def record_component_builds(mmd, module, initial_batch=1,
@@ -876,65 +892,66 @@ def record_component_builds(mmd, module, initial_batch=1,
if main_mmd:
# Check for components that are in both MMDs before merging since MBS
# currently can't handle that situation.
duplicate_components = [rpm for rpm in main_mmd.components.rpms.keys()
if rpm in mmd.components.rpms.keys()]
duplicate_components = [rpm for rpm in main_mmd.get_rpm_components().keys()
if rpm in mmd.get_rpm_components()]
if duplicate_components:
error_msg = (
'The included module "{0}" in "{1}" have the following '
'conflicting components: {2}'
.format(mmd.name, main_mmd.name,
', '.join(duplicate_components)))
'conflicting components: {2}'.format(
mmd.get_name(), main_mmd.get_name(), ', '.join(duplicate_components)))
raise UnprocessableEntity(error_msg)
merge_included_mmd(main_mmd, mmd)
else:
main_mmd = mmd
# If the modulemd yaml specifies components, then submit them for build
if mmd.components:
components = mmd.components.all
components.sort(key=lambda x: x.buildorder)
rpm_components = mmd.get_rpm_components().values()
module_components = mmd.get_module_components().values()
all_components = rpm_components + module_components
if not all_components:
return
weights = module_build_service.builder.GenericBuilder.get_build_weights(
[c.name for c in components])
rpm_weights = module_build_service.builder.GenericBuilder.get_build_weights(
[c.get_name() for c in rpm_components])
all_components.sort(key=lambda x: x.get_buildorder())
# We do not start with batch = 0 here, because the first batch is
# reserved for module-build-macros. First real components must be
# planned for batch 2 and following.
batch = initial_batch
# We do not start with batch = 0 here, because the first batch is
# reserved for module-build-macros. First real components must be
# planned for batch 2 and following.
batch = initial_batch
for component in all_components:
# Increment the batch number when buildorder increases.
if previous_buildorder != component.get_buildorder():
previous_buildorder = component.get_buildorder()
batch += 1
for pkg in components:
# Increment the batch number when buildorder increases.
if previous_buildorder != pkg.buildorder:
previous_buildorder = pkg.buildorder
batch += 1
# If the component is another module, we fetch its modulemd file
# and record its components recursively with the initial_batch
# set to our current batch, so the components of this module
# are built in the right global order.
if isinstance(component, Modulemd.ComponentModule):
full_url = component.get_repository() + "?#" + component.get_ref()
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = _fetch_mmd(full_url, whitelist_url=True)[0]
batch = record_component_builds(included_mmd, module, batch,
previous_buildorder, main_mmd, session=session)
continue
# If the pkg is another module, we fetch its modulemd file
# and record its components recursively with the initial_batch
# set to our current batch, so the components of this module
# are built in the right global order.
if isinstance(pkg, modulemd.ModuleComponentModule):
full_url = pkg.repository + "?#" + pkg.ref
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = _fetch_mmd(full_url, whitelist_url=True)[0]
batch = record_component_builds(included_mmd, module, batch,
previous_buildorder, main_mmd, session=session)
continue
component_ref = mmd.get_xmd()['mbs']['rpms'][component.get_name()]['ref']
full_url = component.get_repository() + "?#" + component_ref
build = models.ComponentBuild(
module_id=module.id,
package=component.get_name(),
format="rpms",
scmurl=full_url,
batch=batch,
ref=component_ref,
weight=rpm_weights[component.get_name()]
)
session.add(build)
pkgref = mmd.xmd['mbs']['rpms'][pkg.name]['ref']
full_url = pkg.repository + "?#" + pkgref
build = models.ComponentBuild(
module_id=module.id,
package=pkg.name,
format="rpms",
scmurl=full_url,
batch=batch,
ref=pkgref,
weight=weights[pkg.name]
)
session.add(build)
return batch
return batch
def submit_module_build_from_yaml(username, handle, stream=None, skiptests=False,
@@ -949,11 +966,13 @@ def submit_module_build_from_yaml(username, handle, stream=None, skiptests=False
dt = datetime.utcfromtimestamp(int(time.time()))
def_name = str(os.path.splitext(os.path.basename(handle.filename))[0])
def_version = int(dt.strftime("%Y%m%d%H%M%S"))
mmd.name = mmd.name or def_name
mmd.stream = stream or mmd.stream or "master"
mmd.version = mmd.version or def_version
mmd.set_name(mmd.get_name() or def_name)
mmd.set_stream(stream or mmd.get_stream() or "master")
mmd.set_version(mmd.get_version() or def_version)
if skiptests:
mmd.buildopts.rpms.macros += "\n\n%__spec_check_pre exit 0\n"
buildopts = mmd.get_rpm_buildopts()
buildopts["macros"] = buildopts.get("macros", "") + "\n\n%__spec_check_pre exit 0\n"
mmd.set_rpm_buildopts(buildopts)
return submit_module_build(username, None, mmd, None, optional_params)
@@ -980,7 +999,7 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
# and fails to import them because of dep-chain.
validate_mmd(mmd)
module = models.ModuleBuild.query.filter_by(
name=mmd.name, stream=mmd.stream, version=str(mmd.version)).first()
name=mmd.get_name(), stream=mmd.get_stream(), version=str(mmd.get_version())).first()
if module:
log.debug('Checking whether module build already exist.')
if module.state != models.BUILD_STATES['failed']:
@@ -1015,9 +1034,9 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
module = models.ModuleBuild.create(
db.session,
conf,
name=mmd.name,
stream=mmd.stream,
version=str(mmd.version),
name=mmd.get_name(),
stream=mmd.get_stream(),
version=str(mmd.get_version()),
modulemd=mmd.dumps(),
scmurl=url,
username=username,
@@ -1027,7 +1046,7 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
db.session.add(module)
db.session.commit()
log.info("%s submitted build of %s, stream=%s, version=%s", username,
mmd.name, mmd.stream, mmd.version)
mmd.get_name(), mmd.get_stream(), mmd.get_version())
return module
@@ -1141,8 +1160,8 @@ def _get_reusable_module(session, module):
# Find the latest module that is in the done or ready state
previous_module_build = session.query(models.ModuleBuild)\
.filter_by(name=mmd.name)\
.filter_by(stream=mmd.stream)\
.filter_by(name=mmd.get_name())\
.filter_by(stream=mmd.get_stream())\
.filter(models.ModuleBuild.state.in_([3, 5]))\
.filter(models.ModuleBuild.scmurl.isnot(None))\
.order_by(models.ModuleBuild.time_completed.desc())
@@ -1268,10 +1287,10 @@ def get_reusable_component(session, module, component_name,
is called to get the ModuleBuild instance. Consider passing the ModuleBuild
instance in case you plan to call get_reusable_component repeatedly for the
same module to make this method faster.
:param mmd: ModuleMetadata of `module`. If not passed, it is taken from
:param mmd: ModuleMd.Module of `module`. If not passed, it is taken from
module.mmd(). Consider passing this arg in case you plan to call
get_reusable_component repeatedly for the same module to make this method faster.
:param old_mmd: ModuleMetadata of `previous_module_build`. If not passed,
:param old_mmd: ModuleMd.Module of `previous_module_build`. If not passed,
it is taken from previous_module_build.mmd(). Consider passing this arg in
case you plan to call get_reusable_component repeatedly for the same
module to make this method faster.
@@ -1334,13 +1353,7 @@ def get_reusable_component(session, module, component_name,
return None
# If the mmd.buildopts.macros.rpms changed, we cannot reuse
modulemd_macros = ""
old_modulemd_macros = ""
if mmd.buildopts and mmd.buildopts.rpms:
modulemd_macros = mmd.buildopts.rpms.macros
if old_mmd.buildopts and old_mmd.buildopts.rpms:
modulemd_macros = old_mmd.buildopts.rpms.macros
if modulemd_macros != old_modulemd_macros:
if mmd.get_rpm_buildopts().get('macros') != old_mmd.get_rpm_buildopts().get('macros'):
log.info('Cannot re-use. Old modulemd macros do not match the new.')
return None