modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-03 13:13:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Port to libmodulemd and support v2 modulemd without module stream expansion

Browse Source
This commit is contained in:
mprahl
2018-02-27 13:29:00 -05:00
parent 329a7acd76
commit f0852d9009
26 changed files with 591 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Jenkinsfile vendored
View File

@@ -26,7 +26,9 @@ node('factory2'){
stage('Pre Setup Node'){
// Install EPEL and the SCLs repo
onmyduffynode 'yum -y install epel-release yum-config-manager centos-release-scl && yum-config-manager --enable rhel-server-rhscl-7-rpms'
onmyduffynode 'yum -y install python27 python-devel rh-python36 @development krb5-devel openssl-devel libffi-devel swig createrepo_c'
onmyduffynode 'yum -y install python27 python-devel rh-python36 @development krb5-devel openssl-devel libffi-devel swig createrepo_c cairo-gobject-devel gobject-introspection-devel'
// Remove this once it's available in the official repos
onmyduffynode 'yum -y install https://kojipkgs.fedoraproject.org//packages/libmodulemd/1.0.4/1.fc26/x86_64/libmodulemd-1.0.4-1.fc26.x86_64.rpm'
// Update pip and setuptools and install tox in the SCL environment
onmyduffynode 'scl enable python27 \'pip install --upgrade pip setuptools tox\''
onmyduffynode 'scl enable rh-python36 \'pip install --upgrade pip setuptools tox\''

4
Vagrantfile vendored
View File

@@ -21,6 +21,7 @@ $script = <<SCRIPT
python-devel \
python-docutils \
python-flask \
python-gobject-base \
python-m2ext \
python-mock \
python-qpid \
@@ -33,7 +34,8 @@ $script = <<SCRIPT
redhat-rpm-config \
rpm-build \
swig \
systemd-devel
systemd-devel \
https://kojipkgs.fedoraproject.org//packages/libmodulemd/1.0.4/1.fc26/x86_64/libmodulemd-1.0.4-1.fc26.x86_64.rpm
cd /opt/module_build_service
python setup.py develop
python setup.py egg_info

16
module_build_service/builder/CoprModuleBuilder.py
View File

@@ -84,12 +84,16 @@ class CoprModuleBuilder(GenericBuilder):
self._create_module_safe()
mmd = self.module.mmd()
# Even though get_buildrequires returns a dictionary with the values as lists, there will
# always be a single item in the list after MBS processes it
buildrequires_d = {name: dep.get()[0]
for name, dep in mmd.get_dependencies()[0].get_buildrequires().items()}
buildrequires = ["@{}:{}/{}".format(n, s, "buildroot")
for n, s in mmd.buildrequires.items()]
for n, s in buildrequires_d.items()]
buildroot_profile = mmd.profiles.get("buildroot")
buildroot_profile = mmd.get_profiles().get("buildroot")
if buildroot_profile:
buildrequires.extend(buildroot_profile.rpms)
buildrequires.extend(buildroot_profile.get_rpms())
self._update_chroot(packages=buildrequires)
@@ -150,9 +154,9 @@ class CoprModuleBuilder(GenericBuilder):
# Write module's name, stream and version into the modulemd file
# so Copr can parse it from there
mmd = self.module.mmd()
mmd.name = str(self.module.name)
mmd.stream = str(self.module.stream)
mmd.version = int(self.module.version)
mmd.set_name(str(self.module.name))
mmd.set_stream(str(self.module.stream))
mmd.set_version(int(self.module.version))
modulemd = tempfile.mktemp()
mmd.dump(modulemd)

7
module_build_service/builder/KojiModuleBuilder.py
View File

@@ -238,7 +238,7 @@ class KojiModuleBuilder(GenericBuilder):
# module-build-macros to conflict with these filtered RPMs to ensure
# they won't be installed to buildroot.
filter_conflicts = ""
for req_name, req_data in mmd.xmd["mbs"]["buildrequires"].items():
for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():
if req_data["filtered_rpms"]:
filter_conflicts += "# Filtered rpms from %s module:\n" % (
req_name)
@@ -300,8 +300,9 @@ chmod 644 %buildroot/%_sysconfdir/rpm/macros.zz-modules
filter_conflicts=filter_conflicts)
modulemd_macros = ""
if mmd.buildopts and mmd.buildopts.rpms:
modulemd_macros = mmd.buildopts.rpms.macros
rpm_buildopts = mmd.get_rpm_buildopts()
if rpm_buildopts:
modulemd_macros = rpm_buildopts.get('macros')
macros_content = """

13
module_build_service/builder/MockModuleBuilder.py
View File

@@ -26,12 +26,14 @@ import logging
import os
import koji
import kobo.rpmlib
import modulemd
import pipes
import platform
import re
import threading
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from module_build_service import conf, log
import module_build_service.scm
import module_build_service.utils
@@ -182,17 +184,20 @@ class MockModuleBuilder(GenericBuilder):
if m1.last_batch_id() == m1.batch:
# If RPM is filtered-out, do not add it to artifacts list.
nvr = kobo.rpmlib.parse_nvr(rpm)
if nvr["name"] in m1_mmd.filter.rpms:
if nvr["name"] in m1_mmd.get_rpm_filter().get():
continue
pkglist_f.write(rpm + '\n')
rpm = rpm[:-len(".rpm")]
m1_mmd.artifacts.add_rpm(str(rpm))
component = Modulemd.ComponentRpm()
component.set_name(str(rpm))
component.set_rationale('none')
m1_mmd.add_rpm_component(component)
pkglist_f.close()
mmd_path = os.path.join(path, "modules.yaml")
modulemd.dump_all(mmd_path, [m1_mmd])
m1_mmd.dump(mmd_path)
# Generate repo and inject modules.yaml there.
execute_cmd(['/usr/bin/createrepo_c', '--pkglist', pkglist, path])

19
module_build_service/migrations/versions/9ca1c166f426_contexts.py
View File

@@ -14,7 +14,9 @@ from alembic import op
import sqlalchemy as sa
# Data migration imports
import modulemd
import gi
gi.require_version('Modulemd', '1.0')
from gi.repository import Modulemd
import hashlib
import json
from collections import OrderedDict
@@ -40,22 +42,23 @@ def upgrade():
for build in connection.execute(modulebuild.select()):
if not build.modulemd:
continue
mmd = modulemd.ModuleMetadata()
try:
mmd.loads(build.modulemd)
mmd = Modulemd.Module().new_from_string(build.modulemd)
mmd.upgrade()
except Exception:
# If the modulemd isn't parseable then skip this build
continue
mbs_xmd = mmd.xmd.get('mbs', {})
mbs_xmd = mmd.get_xmd().get('mbs', {})
contexts = {}
for property_name in ['buildrequires', 'requires']:
# It's possible this module build was built before MBS filled out xmd or before MBS
# filled out the requires in xmd
if property_name not in mbs_xmd:
# filled out the requires in xmd. We also have to use keys because GLib.Variant
# doesn't support `in` directly.
if property_name not in mbs_xmd.keys():
break
mmd_property = getattr(mmd, property_name)
if mbs_xmd[property_name].keys() != mmd_property.keys():
mmd_property = getattr(mmd.get_dependencies()[0], 'get_{0}'.format(property_name))()
if set(mbs_xmd[property_name].keys()) != set(mmd_property.keys()):
break
mmd_formatted_property = {
dep: info['ref'] for dep, info in mbs_xmd[property_name].items()}

23
module_build_service/models.py
View File

@@ -34,12 +34,13 @@ import hashlib
import sqlalchemy
from sqlalchemy.orm import validates, scoped_session, sessionmaker
from flask import has_app_context
import modulemd as _modulemd
from module_build_service import db, log, get_url_for, app, conf
import module_build_service.messaging
from sqlalchemy.orm import lazyload
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
# Just like koji.BUILD_STATES, except our own codes for modules.
BUILD_STATES = {
@@ -241,9 +242,9 @@ class ModuleBuild(MBSBase):
]
def mmd(self):
mmd = _modulemd.ModuleMetadata()
try:
mmd.loads(self.modulemd)
mmd = Modulemd.Module().new_from_string(self.modulemd)
mmd.upgrade()
except Exception:
raise ValueError("Invalid modulemd")
return mmd
@@ -281,14 +282,18 @@ class ModuleBuild(MBSBase):
@staticmethod
def contexts_from_mmd(mmd_str):
mmd = _modulemd.ModuleMetadata()
mmd.loads(mmd_str)
mbs_xmd = mmd.xmd.get('mbs', {})
try:
mmd = Modulemd.Module().new_from_string(mmd_str)
mmd.upgrade()
except Exception:
raise ValueError("Invalid modulemd")
mbs_xmd = mmd.get_xmd().get('mbs', {})
rv = []
for property_name in ['buildrequires', 'requires']:
if property_name not in mbs_xmd:
# We have to use keys because GLib.Variant doesn't support `in` directly.
if property_name not in mbs_xmd.keys():
raise ValueError('The module\'s modulemd hasn\'t been formatted by MBS')
mmd_property = getattr(mmd, property_name)
mmd_property = getattr(mmd.get_dependencies()[0], 'get_{0}'.format(property_name))()
if set(mbs_xmd[property_name].keys()) != set(mmd_property.keys()):
raise ValueError('The dependencies.{0} section of the modulemd doesn\'t match '
'what is in xmd'.format(property_name))

63
module_build_service/resolver/PDCResolver.py
View File

@@ -26,7 +26,9 @@
"""PDC handler functions."""
import modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
import pdc_client
from module_build_service import db
from module_build_service import models
@@ -99,7 +101,7 @@ class PDCResolver(GenericResolver):
return True
def is_modulemd(data):
return isinstance(data, modulemd.ModuleMetadata)
return isinstance(data, Modulemd.Module)
def is_module_str(data):
return isinstance(data, six.string_types)
@@ -110,14 +112,11 @@ class PDCResolver(GenericResolver):
result = self._variant_dict_from_str(data)
elif is_modulemd(data):
result = {'variant_id': data.name}
# Check if this is an old modulemd that doesn't use the new nomenclature
if hasattr(data, 'release'):
result['variant_release'] = data.release
result['variant_version'] = data.version
else:
result['variant_release'] = data.version
result['variant_version'] = data.stream
result = {
'variant_id': data.get_name(),
'variant_release': data.get_version(),
'variant_version': data.get_stream()
}
elif is_variant_dict(data):
result = data.copy()
@@ -308,11 +307,17 @@ class PDCResolver(GenericResolver):
# module was built. But we still should fallback to plain mmd.requires
# in case this module depends on some older module for which we did
# not populate mmd.xmd['mbs']['requires'].
if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('requires'):
requires = mmd.xmd['mbs']['requires']
requires = {name: data['stream'] for name, data in requires.items()}
mbs_xmd = mmd.get_xmd().get('mbs')
if 'requires' in mbs_xmd.keys():
requires = {name: data['stream'] for name, data in mbs_xmd['requires'].items()}
else:
requires = mmd.requires
# Since MBS doesn't support v2 modulemds submitted by a user, we will
# always only have one stream per require. That way it's safe to just take the first
# element of the list.
# TODO: Change this once module stream expansion is implemented
requires = {
name: deps.get()[0]
for name, deps in mmd.get_dependencies()[0].get_requires().items()}
for name, stream in requires.items():
modified_dep = {
@@ -344,7 +349,7 @@ class PDCResolver(GenericResolver):
results = {}
for key in keys:
results[key] = set()
for module_name, module_info in mmd.xmd['mbs']['buildrequires'].items():
for module_name, module_info in mmd.get_xmd()['mbs']['buildrequires'].items():
local_modules = models.ModuleBuild.local_modules(
db.session, module_name, module_info['stream'])
if local_modules:
@@ -353,8 +358,8 @@ class PDCResolver(GenericResolver):
local_module)
dep_mmd = local_module.mmd()
for key in keys:
if key in dep_mmd.profiles:
results[key] |= dep_mmd.profiles[key].rpms
if key in dep_mmd.get_profiles().keys():
results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())
continue
# Find the dep in the built modules in PDC
@@ -368,11 +373,10 @@ class PDCResolver(GenericResolver):
for module in modules:
yaml = module['modulemd']
dep_mmd = self.extract_modulemd(yaml)
# Take note of what rpms are in this dep's profile.
for key in keys:
if key in dep_mmd.profiles:
results[key] |= dep_mmd.profiles[key].rpms
if key in dep_mmd.get_profiles().keys():
results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())
# Return the union of all rpms in all profiles of the given keys.
return results
@@ -398,20 +402,20 @@ class PDCResolver(GenericResolver):
# This is the set we're going to build up and return.
module_tags = {}
if not isinstance(module_info, modulemd.ModuleMetadata):
if not isinstance(module_info, Modulemd.Module):
queried_module = self._get_module(module_info, strict=strict)
yaml = queried_module['modulemd']
queried_mmd = self.extract_modulemd(yaml, strict=strict)
else:
queried_mmd = module_info
if (not queried_mmd or 'mbs' not in queried_mmd.xmd or
'buildrequires' not in queried_mmd.xmd['mbs']):
if (not queried_mmd or not queried_mmd.get_xmd().get('mbs') or
'buildrequires' not in queried_mmd.get_xmd()['mbs'].keys()):
raise RuntimeError(
'The module "{0!r}" did not contain its modulemd or did not have '
'its xmd attribute filled out in PDC'.format(module_info))
buildrequires = queried_mmd.xmd['mbs']['buildrequires']
buildrequires = queried_mmd.get_xmd()['mbs']['buildrequires']
# Queue up the next tier of deps that we should look at..
for name, details in buildrequires.items():
modified_dep = {
@@ -482,17 +486,16 @@ class PDCResolver(GenericResolver):
filtered_rpms = []
module = self._get_module(module_info, strict=True)
if module.get('modulemd'):
mmd = modulemd.ModuleMetadata()
mmd.loads(module['modulemd'])
if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('commit'):
commit_hash = mmd.xmd['mbs']['commit']
mmd = self.extract_modulemd(module['modulemd'])
if mmd.get_xmd().get('mbs') and 'commit' in mmd.get_xmd()['mbs'].keys():
commit_hash = mmd.get_xmd()['mbs']['commit']
# Find out the particular NVR of filtered packages
if "rpms" in module and mmd.filter and mmd.filter.rpms:
if "rpms" in module and mmd.get_rpm_filter().get():
for rpm in module["rpms"]:
nvr = kobo.rpmlib.parse_nvra(rpm)
# If the package is not filtered, continue
if not nvr["name"] in mmd.filter.rpms:
if not nvr["name"] in mmd.get_rpm_filter().get():
continue
# If the nvr is already in filtered_rpms, continue

11
module_build_service/resolver/base.py
View File

@@ -26,8 +26,10 @@
import six
from abc import ABCMeta, abstractmethod
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
import modulemd
import module_build_service.config as cfg
from module_build_service import conf
@@ -88,8 +90,11 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
@staticmethod
def extract_modulemd(yaml, strict=False):
mmd = modulemd.ModuleMetadata()
mmd.loads(yaml)
try:
mmd = Modulemd.Module().new_from_string(yaml)
mmd.upgrade()
except Exception:
raise ValueError('Invalid modulemd')
return mmd
@abstractmethod

6
module_build_service/scheduler/handlers/modules.py
View File

@@ -146,7 +146,7 @@ def init(config, session, msg):
mmd = build.mmd()
record_component_builds(mmd, build, session=session)
build.build_context, build.runtime_context = build.contexts_from_mmd(mmd.dumps())
mmd.context = build.context
mmd.set_context(build.context)
build.modulemd = mmd.dumps()
build.transition(conf, models.BUILD_STATES["wait"])
# Catch custom exceptions that we can expose to the user
@@ -181,7 +181,7 @@ def wait(config, session, msg):
@module_build_service.utils.retry(interval=10, timeout=120, wait_on=RuntimeError)
def _get_build_containing_xmd_for_mbs():
build = models.ModuleBuild.from_module_event(session, msg)
if 'mbs' in build.mmd().xmd:
if 'mbs' in build.mmd().get_xmd():
return build
session.expire(build)
raise RuntimeError("{!r} doesn't contain xmd information for MBS."
@@ -244,7 +244,7 @@ def wait(config, session, msg):
# Find out the name of Koji tag to which the module's Content
# Generator build should be tagged once the build finishes.
module_names_streams = {mmd.name: mmd.stream
module_names_streams = {mmd.get_name(): mmd.get_stream()
for mmd in deps_dict.values()}
for base_module_name in conf.base_module_names:
if base_module_name in module_names_streams:

309
module_build_service/utils.py
View File

@@ -33,12 +33,13 @@ import kobo.rpmlib
import inspect
import hashlib
from functools import wraps
import modulemd
import yaml
from flask import request, url_for, Response
from datetime import datetime
from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from module_build_service import log, models
from module_build_service.errors import (ValidationError, UnprocessableEntity,
@@ -48,6 +49,7 @@ from module_build_service.errors import (Forbidden, Conflict)
import module_build_service.messaging
from multiprocessing.dummy import Pool as ThreadPool
import module_build_service.resolver
from module_build_service import glib
import concurrent.futures
@@ -558,7 +560,6 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False):
# and fails to import them because of dep-chain.
import module_build_service.scm
yaml = ""
td = None
scm = None
try:
@@ -571,9 +572,7 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False):
scm.checkout(td)
scm.verify()
cofn = scm.get_module_yaml()
with open(cofn, "r") as mmdfile:
yaml = mmdfile.read()
mmd = load_mmd(cofn, is_file=True)
finally:
try:
if td is not None:
@@ -583,45 +582,65 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False):
"Failed to remove temporary directory {!r}: {}".format(
td, str(e)))
mmd = load_mmd(yaml)
# If the name was set in the modulemd, make sure it matches what the scmurl
# says it should be
if mmd.name and mmd.name != scm.name:
if mmd.get_name() and mmd.get_name() != scm.name:
raise ValidationError('The name "{0}" that is stored in the modulemd '
'is not valid'.format(mmd.name))
'is not valid'.format(mmd.get_name()))
else:
mmd.name = scm.name
mmd.set_name(scm.name)
# If the stream was set in the modulemd, make sure it matches what the repo
# branch is
if mmd.stream and mmd.stream != scm.branch:
if mmd.get_stream() and mmd.get_stream() != scm.branch:
raise ValidationError('The stream "{0}" that is stored in the modulemd '
'does not match the branch "{1}"'.format(
mmd.stream, scm.branch))
mmd.get_stream(), scm.branch))
else:
mmd.stream = str(scm.branch)
mmd.set_stream(str(scm.branch))
# If the version is in the modulemd, throw an exception since the version
# is generated by pdc-updater
if mmd.version:
if mmd.get_version():
raise ValidationError('The version "{0}" is already defined in the '
'modulemd but it shouldn\'t be since the version '
'is generated based on the commit time'.format(
mmd.version))
mmd.get_version()))
else:
mmd.version = int(scm.version)
mmd.set_version(int(scm.version))
return mmd, scm
def load_mmd(yaml):
mmd = modulemd.ModuleMetadata()
def load_mmd(yaml, is_file=False):
try:
mmd.loads(yaml)
except Exception as e:
log.error('Invalid modulemd: %s' % str(e))
raise UnprocessableEntity('Invalid modulemd: %s' % str(e))
if is_file:
mmd = Modulemd.Module().new_from_file(yaml)
else:
mmd = Modulemd.Module().new_from_string(yaml)
# If the modulemd was v1, it will be upgraded to v2
mmd.upgrade()
except Exception:
error = 'The following invalid modulemd was encountered: {0}'.format(yaml)
log.error(error)
raise UnprocessableEntity(error)
# MBS doesn't support module stream expansion yet but supports the v2 modulemd format,
# so if module stream expansion syntax is used, fail the submission
# TODO: Once module stream expansion is supported, the get_dependencies() function should
# be squashed to a single list on resulting modulemds
error_msg = 'Module stream expansion is not yet supported in MBS'
deps_list = mmd.get_dependencies()
if len(deps_list) > 1:
raise UnprocessableEntity(error_msg)
elif len(deps_list) == 1:
for dep_type in ['requires', 'buildrequires']:
deps = getattr(deps_list[0], 'get_{0}'.format(dep_type))()
for streams in deps.values():
if len(streams.get()) != 1:
raise UnprocessableEntity(error_msg)
elif streams.get()[0].startswith('-'):
raise UnprocessableEntity(error)
return mmd
@@ -632,15 +651,14 @@ def _scm_get_latest(pkg):
# to the specific commit available at the time of
# submission (now).
pkgref = module_build_service.scm.SCM(
pkg.repository).get_latest(pkg.ref)
pkg.get_repository()).get_latest(pkg.get_ref())
except Exception as e:
log.exception(e)
return {
'error': "Failed to get the latest commit for %s#%s" % (pkg.repository, pkg.ref)
}
return {'error': "Failed to get the latest commit for %s#%s" % (
pkg.get_repository(), pkg.get_ref())}
return {
'pkg_name': pkg.name,
'pkg_name': pkg.get_name(),
'pkg_ref': pkgref,
'error': None
}
@@ -711,19 +729,15 @@ def load_local_builds(local_build_nsvs, session=None):
# Load the modulemd metadata.
path = os.path.join(conf.mock_resultsdir, found_build[3], 'results')
mmd_path = os.path.join(path, 'modules.yaml')
with open(mmd_path, 'r') as f:
mmd_data = yaml.safe_load(f)
mmd = modulemd.ModuleMetadata()
mmd.loadd(mmd_data)
mmd = load_mmd(os.path.join(path, 'modules.yaml'), is_file=True)
# Create ModuleBuild in database.
module = models.ModuleBuild.create(
session,
conf,
name=mmd.name,
stream=mmd.stream,
version=str(mmd.version),
name=mmd.get_name(),
stream=mmd.get_stream(),
version=str(mmd.get_version()),
modulemd=mmd.dumps(),
scmurl="",
username="mbs",
@@ -755,7 +769,8 @@ def format_mmd(mmd, scmurl, session=None):
if not session:
session = db.session
mmd.xmd['mbs'] = {'scmurl': scmurl, 'commit': None}
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs'] = {'scmurl': scmurl or '', 'commit': ''}
local_modules = models.ModuleBuild.local_modules(session)
local_modules = {m.name + "-" + m.stream: m for m in local_modules}
@@ -771,62 +786,56 @@ def format_mmd(mmd, scmurl, session=None):
else:
full_scm_hash = scm.get_full_commit_hash()
mmd.xmd['mbs']['commit'] = full_scm_hash
xmd['mbs']['commit'] = full_scm_hash
# If a commit hash wasn't provided then just get the latest from master
else:
mmd.xmd['mbs']['commit'] = scm.get_latest()
xmd['mbs']['commit'] = scm.get_latest()
resolver = module_build_service.resolver.GenericResolver.create(conf)
# Resolve Build-requires.
if mmd.buildrequires:
mmd.xmd['mbs']['buildrequires'] = resolver.resolve_requires(
mmd.buildrequires)
else:
mmd.xmd['mbs']['buildrequires'] = {}
# Resolve buildrequires and requires
# Reformat the input for resolve_requires to match the old modulemd format
dep_obj = mmd.get_dependencies()[0]
br_dict = {br: br_list.get()[0] for br, br_list in dep_obj.get_buildrequires().items()}
req_dict = {req: req_list.get()[0] for req, req_list in dep_obj.get_requires().items()}
xmd['mbs']['buildrequires'] = resolver.resolve_requires(br_dict)
xmd['mbs']['requires'] = resolver.resolve_requires(req_dict)
# Resolve Requires.
if mmd.requires:
mmd.xmd['mbs']['requires'] = resolver.resolve_requires(mmd.requires)
else:
mmd.xmd['mbs']['requires'] = {}
if mmd.components:
if 'rpms' not in mmd.xmd['mbs']:
mmd.xmd['mbs']['rpms'] = {}
if mmd.get_rpm_components() or mmd.get_module_components():
if 'rpms' not in xmd['mbs']:
xmd['mbs']['rpms'] = {}
# Add missing data in RPM components
for pkgname, pkg in mmd.components.rpms.items():
if pkg.repository and not conf.rpms_allow_repository:
for pkgname, pkg in mmd.get_rpm_components().items():
if pkg.get_repository() and not conf.rpms_allow_repository:
raise Forbidden(
"Custom component repositories aren't allowed. "
"%r bears repository %r" % (pkgname, pkg.repository))
if pkg.cache and not conf.rpms_allow_cache:
"%r bears repository %r" % (pkgname, pkg.get_repository()))
if pkg.get_cache() and not conf.rpms_allow_cache:
raise Forbidden(
"Custom component caches aren't allowed. "
"%r bears cache %r" % (pkgname, pkg.cache))
if not pkg.repository:
pkg.repository = conf.rpms_default_repository + pkgname
if not pkg.cache:
pkg.cache = conf.rpms_default_cache + pkgname
if not pkg.ref:
pkg.ref = 'master'
if not pkg.get_repository():
pkg.set_repository(conf.rpms_default_repository + pkgname)
if not pkg.get_cache():
pkg.set_cache(conf.rpms_default_cache + pkgname)
if not pkg.get_ref():
pkg.set_ref('master')
# Add missing data in included modules components
for modname, mod in mmd.components.modules.items():
if mod.repository and not conf.modules_allow_repository:
for modname, mod in mmd.get_module_components().items():
if mod.get_repository() and not conf.modules_allow_repository:
raise Forbidden(
"Custom module repositories aren't allowed. "
"%r bears repository %r" % (modname, mod.repository))
if not mod.repository:
mod.repository = conf.modules_default_repository + modname
if not mod.ref:
mod.ref = 'master'
"%r bears repository %r" % (modname, mod.get_repository()))
if not mod.get_repository():
mod.set_repository(conf.modules_default_repository + modname)
if not mod.get_ref():
mod.set_ref('master')
# Check that SCM URL is valid and replace potential branches in
# pkg.ref by real SCM hash and store the result to our private xmd
# place in modulemd.
# Check that SCM URL is valid and replace potential branches in pkg refs
# by real SCM hash and store the result to our private xmd place in modulemd.
pool = ThreadPool(20)
pkg_dicts = pool.map(_scm_get_latest, mmd.components.rpms.values())
pkg_dicts = pool.map(_scm_get_latest, mmd.get_rpm_components().values())
err_msg = ""
for pkg_dict in pkg_dicts:
if pkg_dict["error"]:
@@ -834,17 +843,20 @@ def format_mmd(mmd, scmurl, session=None):
else:
pkg_name = pkg_dict["pkg_name"]
pkg_ref = pkg_dict["pkg_ref"]
mmd.xmd['mbs']['rpms'][pkg_name] = {'ref': pkg_ref}
xmd['mbs']['rpms'][pkg_name] = {'ref': pkg_ref}
if err_msg:
raise UnprocessableEntity(err_msg)
# Set the modified xmd back to the modulemd
mmd.set_xmd(glib.dict_values(xmd))
def validate_mmd(mmd):
for modname, mod in mmd.components.modules.items():
if mod.repository and not conf.modules_allow_repository:
for modname, mod in mmd.get_module_components().items():
if mod.get_repository() and not conf.modules_allow_repository:
raise Forbidden(
"Custom module repositories aren't allowed. "
"%r bears repository %r" % (modname, mod.repository))
"%r bears repository %r" % (modname, mod.get_repository()))
def merge_included_mmd(mmd, included_mmd):
@@ -852,11 +864,15 @@ def merge_included_mmd(mmd, included_mmd):
Merges two modulemds. This merges only metadata which are needed in
the `main` when it includes another module defined by `included_mmd`
"""
if 'rpms' in included_mmd.xmd['mbs']:
if 'rpms' not in mmd.xmd['mbs']:
mmd.xmd['mbs']['rpms'] = included_mmd.xmd['mbs']['rpms']
included_xmd = glib.from_variant_dict(included_mmd.get_xmd())
if 'rpms' in included_xmd['mbs']:
xmd = glib.from_variant_dict(mmd.get_xmd())
if 'rpms' not in xmd['mbs']:
xmd['mbs']['rpms'] = included_xmd['mbs']['rpms']
else:
mmd.xmd['mbs']['rpms'].update(included_mmd.xmd['mbs']['rpms'])
xmd['mbs']['rpms'].update(included_xmd['mbs']['rpms'])
# Set the modified xmd back to the modulemd
mmd.set_xmd(glib.dict_values(xmd))
def record_component_builds(mmd, module, initial_batch=1,
@@ -876,65 +892,66 @@ def record_component_builds(mmd, module, initial_batch=1,
if main_mmd:
# Check for components that are in both MMDs before merging since MBS
# currently can't handle that situation.
duplicate_components = [rpm for rpm in main_mmd.components.rpms.keys()
if rpm in mmd.components.rpms.keys()]
duplicate_components = [rpm for rpm in main_mmd.get_rpm_components().keys()
if rpm in mmd.get_rpm_components()]
if duplicate_components:
error_msg = (
'The included module "{0}" in "{1}" have the following '
'conflicting components: {2}'
.format(mmd.name, main_mmd.name,
', '.join(duplicate_components)))
'conflicting components: {2}'.format(
mmd.get_name(), main_mmd.get_name(), ', '.join(duplicate_components)))
raise UnprocessableEntity(error_msg)
merge_included_mmd(main_mmd, mmd)
else:
main_mmd = mmd
# If the modulemd yaml specifies components, then submit them for build
if mmd.components:
components = mmd.components.all
components.sort(key=lambda x: x.buildorder)
rpm_components = mmd.get_rpm_components().values()
module_components = mmd.get_module_components().values()
all_components = rpm_components + module_components
if not all_components:
return
weights = module_build_service.builder.GenericBuilder.get_build_weights(
[c.name for c in components])
rpm_weights = module_build_service.builder.GenericBuilder.get_build_weights(
[c.get_name() for c in rpm_components])
all_components.sort(key=lambda x: x.get_buildorder())
# We do not start with batch = 0 here, because the first batch is
# reserved for module-build-macros. First real components must be
# planned for batch 2 and following.
batch = initial_batch
# We do not start with batch = 0 here, because the first batch is
# reserved for module-build-macros. First real components must be
# planned for batch 2 and following.
batch = initial_batch
for component in all_components:
# Increment the batch number when buildorder increases.
if previous_buildorder != component.get_buildorder():
previous_buildorder = component.get_buildorder()
batch += 1
for pkg in components:
# Increment the batch number when buildorder increases.
if previous_buildorder != pkg.buildorder:
previous_buildorder = pkg.buildorder
batch += 1
# If the component is another module, we fetch its modulemd file
# and record its components recursively with the initial_batch
# set to our current batch, so the components of this module
# are built in the right global order.
if isinstance(component, Modulemd.ComponentModule):
full_url = component.get_repository() + "?#" + component.get_ref()
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = _fetch_mmd(full_url, whitelist_url=True)[0]
batch = record_component_builds(included_mmd, module, batch,
previous_buildorder, main_mmd, session=session)
continue
# If the pkg is another module, we fetch its modulemd file
# and record its components recursively with the initial_batch
# set to our current batch, so the components of this module
# are built in the right global order.
if isinstance(pkg, modulemd.ModuleComponentModule):
full_url = pkg.repository + "?#" + pkg.ref
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = _fetch_mmd(full_url, whitelist_url=True)[0]
batch = record_component_builds(included_mmd, module, batch,
previous_buildorder, main_mmd, session=session)
continue
component_ref = mmd.get_xmd()['mbs']['rpms'][component.get_name()]['ref']
full_url = component.get_repository() + "?#" + component_ref
build = models.ComponentBuild(
module_id=module.id,
package=component.get_name(),
format="rpms",
scmurl=full_url,
batch=batch,
ref=component_ref,
weight=rpm_weights[component.get_name()]
)
session.add(build)
pkgref = mmd.xmd['mbs']['rpms'][pkg.name]['ref']
full_url = pkg.repository + "?#" + pkgref
build = models.ComponentBuild(
module_id=module.id,
package=pkg.name,
format="rpms",
scmurl=full_url,
batch=batch,
ref=pkgref,
weight=weights[pkg.name]
)
session.add(build)
return batch
return batch
def submit_module_build_from_yaml(username, handle, stream=None, skiptests=False,
@@ -949,11 +966,13 @@ def submit_module_build_from_yaml(username, handle, stream=None, skiptests=False
dt = datetime.utcfromtimestamp(int(time.time()))
def_name = str(os.path.splitext(os.path.basename(handle.filename))[0])
def_version = int(dt.strftime("%Y%m%d%H%M%S"))
mmd.name = mmd.name or def_name
mmd.stream = stream or mmd.stream or "master"
mmd.version = mmd.version or def_version
mmd.set_name(mmd.get_name() or def_name)
mmd.set_stream(stream or mmd.get_stream() or "master")
mmd.set_version(mmd.get_version() or def_version)
if skiptests:
mmd.buildopts.rpms.macros += "\n\n%__spec_check_pre exit 0\n"
buildopts = mmd.get_rpm_buildopts()
buildopts["macros"] = buildopts.get("macros", "") + "\n\n%__spec_check_pre exit 0\n"
mmd.set_rpm_buildopts(buildopts)
return submit_module_build(username, None, mmd, None, optional_params)
@@ -980,7 +999,7 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
# and fails to import them because of dep-chain.
validate_mmd(mmd)
module = models.ModuleBuild.query.filter_by(
name=mmd.name, stream=mmd.stream, version=str(mmd.version)).first()
name=mmd.get_name(), stream=mmd.get_stream(), version=str(mmd.get_version())).first()
if module:
log.debug('Checking whether module build already exist.')
if module.state != models.BUILD_STATES['failed']:
@@ -1015,9 +1034,9 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
module = models.ModuleBuild.create(
db.session,
conf,
name=mmd.name,
stream=mmd.stream,
version=str(mmd.version),
name=mmd.get_name(),
stream=mmd.get_stream(),
version=str(mmd.get_version()),
modulemd=mmd.dumps(),
scmurl=url,
username=username,
@@ -1027,7 +1046,7 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
db.session.add(module)
db.session.commit()
log.info("%s submitted build of %s, stream=%s, version=%s", username,
mmd.name, mmd.stream, mmd.version)
mmd.get_name(), mmd.get_stream(), mmd.get_version())
return module
@@ -1141,8 +1160,8 @@ def _get_reusable_module(session, module):
# Find the latest module that is in the done or ready state
previous_module_build = session.query(models.ModuleBuild)\
.filter_by(name=mmd.name)\
.filter_by(stream=mmd.stream)\
.filter_by(name=mmd.get_name())\
.filter_by(stream=mmd.get_stream())\
.filter(models.ModuleBuild.state.in_([3, 5]))\
.filter(models.ModuleBuild.scmurl.isnot(None))\
.order_by(models.ModuleBuild.time_completed.desc())
@@ -1268,10 +1287,10 @@ def get_reusable_component(session, module, component_name,
is called to get the ModuleBuild instance. Consider passing the ModuleBuild
instance in case you plan to call get_reusable_component repeatedly for the
same module to make this method faster.
:param mmd: ModuleMetadata of `module`. If not passed, it is taken from
:param mmd: ModuleMd.Module of `module`. If not passed, it is taken from
module.mmd(). Consider passing this arg in case you plan to call
get_reusable_component repeatedly for the same module to make this method faster.
:param old_mmd: ModuleMetadata of `previous_module_build`. If not passed,
:param old_mmd: ModuleMd.Module of `previous_module_build`. If not passed,
it is taken from previous_module_build.mmd(). Consider passing this arg in
case you plan to call get_reusable_component repeatedly for the same
module to make this method faster.
@@ -1334,13 +1353,7 @@ def get_reusable_component(session, module, component_name,
return None
# If the mmd.buildopts.macros.rpms changed, we cannot reuse
modulemd_macros = ""
old_modulemd_macros = ""
if mmd.buildopts and mmd.buildopts.rpms:
modulemd_macros = mmd.buildopts.rpms.macros
if old_mmd.buildopts and old_mmd.buildopts.rpms:
modulemd_macros = old_mmd.buildopts.rpms.macros
if modulemd_macros != old_modulemd_macros:
if mmd.get_rpm_buildopts().get('macros') != old_mmd.get_rpm_buildopts().get('macros'):
log.info('Cannot re-use. Old modulemd macros do not match the new.')
return None

2
requirements.txt
View File

@@ -14,13 +14,13 @@ koji
ldap3
m2crypto
m2ext
modulemd>=1.3.3,<2.0.0
moksha.hub
munch
pdc-client
psutil
pyOpenSSL
python-fedora
pygobject
requests
requests_kerberos # Client only
six

91
tests/__init__.py
View File

@@ -26,13 +26,16 @@ from mock import patch
import time
from traceback import extract_stack
import modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
import koji
import module_build_service
from module_build_service import db
from module_build_service.utils import get_rpm_release
from module_build_service.config import init_config
from module_build_service.models import ModuleBuild, ComponentBuild, make_session, BUILD_STATES
from module_build_service import glib
base_dir = os.path.dirname(__file__)
@@ -271,9 +274,9 @@ def scheduler_init_data(tangerine_state=None):
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(
current_dir, 'staged_data', 'formatted_testmodule.yaml')
mmd = modulemd.ModuleMetadata()
mmd.load(formatted_testmodule_yml_path)
mmd.components.rpms['tangerine'].buildorder = 0
mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path)
mmd.upgrade()
mmd.get_rpm_components()['tangerine'].set_buildorder(0)
build_one = module_build_service.models.ModuleBuild()
build_one.name = 'testmodule'
@@ -376,8 +379,8 @@ def reuse_component_init_data():
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(
current_dir, 'staged_data', 'formatted_testmodule.yaml')
mmd = modulemd.ModuleMetadata()
mmd.load(formatted_testmodule_yml_path)
mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path)
mmd.upgrade()
build_one = module_build_service.models.ModuleBuild()
build_one.name = 'testmodule'
@@ -395,9 +398,11 @@ def reuse_component_init_data():
build_one.time_completed = datetime(2017, 2, 15, 16, 19, 35)
build_one.rebuild_strategy = 'changed-and-after'
build_one_component_release = get_rpm_release(build_one)
mmd.version = build_one.version
mmd.xmd['mbs']['scmurl'] = build_one.scmurl
mmd.xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba'
mmd.set_version(build_one.version)
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['scmurl'] = build_one.scmurl
xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba'
mmd.set_xmd(glib.dict_values(xmd))
build_one.modulemd = mmd.dumps()
component_one_build_one = module_build_service.models.ComponentBuild()
@@ -475,9 +480,11 @@ def reuse_component_init_data():
build_two.time_modified = datetime(2017, 2, 19, 16, 8, 18)
build_two.rebuild_strategy = 'changed-and-after'
build_two_component_release = get_rpm_release(build_two)
mmd.version = build_one.version
mmd.xmd['mbs']['scmurl'] = build_one.scmurl
mmd.xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8'
mmd.set_version(build_one.version)
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['scmurl'] = build_one.scmurl
xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8'
mmd.set_xmd(glib.dict_values(xmd))
build_two.modulemd = mmd.dumps()
component_one_build_two = module_build_service.models.ComponentBuild()
@@ -540,25 +547,22 @@ def reuse_shared_userspace_init_data():
clean_database()
with make_session(conf) as session:
mmd = modulemd.ModuleMetadata()
# Create shared-userspace-570, state is COMPLETE, all components
# are properly built.
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(
current_dir, 'staged_data', 'shared-userspace-570.yaml')
with open(formatted_testmodule_yml_path, 'r') as f:
yaml = f.read()
mmd.loads(yaml)
mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path)
mmd.upgrade()
build_one = module_build_service.models.ModuleBuild()
build_one.name = mmd.name
build_one.stream = mmd.stream
build_one.version = mmd.version
build_one.name = mmd.get_name()
build_one.stream = mmd.get_stream()
build_one.version = mmd.get_version()
build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe'
build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a'
build_one.state = BUILD_STATES['ready']
build_one.modulemd = yaml
build_one.modulemd = mmd.dumps()
build_one.koji_tag = 'module-testmodule-master-20170109091357'
build_one.scmurl = ('git://pkgs.stg.fedoraproject.org/modules/testmodule.'
'git?#7fea453')
@@ -571,21 +575,21 @@ def reuse_shared_userspace_init_data():
session.add(build_one)
components = mmd.components.all
components.sort(key=lambda x: x.buildorder)
components = mmd.get_rpm_components().values()
components.sort(key=lambda x: x.get_buildorder())
previous_buildorder = None
batch = 1
for pkg in components:
# Increment the batch number when buildorder increases.
if previous_buildorder != pkg.buildorder:
previous_buildorder = pkg.buildorder
if previous_buildorder != pkg.get_buildorder():
previous_buildorder = pkg.get_buildorder()
batch += 1
pkgref = mmd.xmd['mbs']['rpms'][pkg.name]['ref']
full_url = pkg.repository + "?#" + pkgref
pkgref = mmd.get_xmd()['mbs']['rpms'][pkg.get_name()]['ref']
full_url = pkg.get_repository() + "?#" + pkgref
build = module_build_service.models.ComponentBuild(
module_id=1,
package=pkg.name,
package=pkg.get_name(),
format="rpms",
scmurl=full_url,
batch=batch,
@@ -599,18 +603,17 @@ def reuse_shared_userspace_init_data():
# Create shared-userspace-577, state is WAIT, no component built
formatted_testmodule_yml_path = os.path.join(
current_dir, 'staged_data', 'shared-userspace-577.yaml')
with open(formatted_testmodule_yml_path, 'r') as f:
yaml = f.read()
mmd.loads(yaml)
mmd2 = Modulemd.Module().new_from_file(formatted_testmodule_yml_path)
mmd2.upgrade()
build_one = module_build_service.models.ModuleBuild()
build_one.name = mmd.name
build_one.stream = mmd.stream
build_one.version = mmd.version
build_one.name = mmd2.get_name()
build_one.stream = mmd2.get_stream()
build_one.version = mmd2.get_version()
build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe'
build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a'
build_one.state = BUILD_STATES['done']
build_one.modulemd = yaml
build_one.modulemd = mmd2.dumps()
build_one.koji_tag = 'module-testmodule-master-20170109091357'
build_one.scmurl = ('git://pkgs.stg.fedoraproject.org/modules/testmodule.'
'git?#7fea453')
@@ -623,24 +626,24 @@ def reuse_shared_userspace_init_data():
session.add(build_one)
components = mmd.components.all
components2 = mmd2.get_rpm_components().values()
# Store components to database in different order than for 570 to
# reproduce the reusing issue.
components.sort(key=lambda x: len(x.name))
components.sort(key=lambda x: x.buildorder)
components2.sort(key=lambda x: len(x.get_name()))
components2.sort(key=lambda x: x.get_buildorder())
previous_buildorder = None
batch = 1
for pkg in components:
for pkg in components2:
# Increment the batch number when buildorder increases.
if previous_buildorder != pkg.buildorder:
previous_buildorder = pkg.buildorder
if previous_buildorder != pkg.get_buildorder():
previous_buildorder = pkg.get_buildorder()
batch += 1
pkgref = mmd.xmd['mbs']['rpms'][pkg.name]['ref']
full_url = pkg.repository + "?#" + pkgref
pkgref = mmd2.get_xmd()['mbs']['rpms'][pkg.get_name()]['ref']
full_url = pkg.get_repository() + "?#" + pkgref
build = module_build_service.models.ComponentBuild(
module_id=2,
package=pkg.name,
package=pkg.get_name(),
format="rpms",
scmurl=full_url,
batch=batch,

49
tests/conftest.py
View File

@@ -27,15 +27,25 @@ import six
import pytest
import mock
import pdc_client.test_helpers
import modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from module_build_service import glib
BASE_DIR = os.path.dirname(__file__)
STAGED_DATA_DIR = os.path.join(BASE_DIR, 'staged_data')
with open(os.path.join(STAGED_DATA_DIR, 'platform.yaml')) as f:
PLATFORM_MODULEMD = f.read()
with open(os.path.join(STAGED_DATA_DIR, 'formatted_testmodule.yaml')) as f:
TESTMODULE_MODULEMD = f.read()
_mmd = Modulemd.Module().new_from_file(
os.path.join(STAGED_DATA_DIR, 'platform.yaml'))
_mmd.upgrade()
PLATFORM_MODULEMD = _mmd.dumps()
_mmd2 = Modulemd.Module().new_from_file(
os.path.join(STAGED_DATA_DIR, 'formatted_testmodule.yaml'))
_mmd2.upgrade()
TESTMODULE_MODULEMD = _mmd2.dumps()
class MockPDCFilterAPI(pdc_client.test_helpers.MockAPI):
@@ -172,27 +182,32 @@ def pdc_module_active(pdc_module_inactive):
def pdc_module_reuse(pdc_module_active):
# Rename it for clarity
pdc_module_reuse = pdc_module_active
mmd = modulemd.ModuleMetadata()
mmd.loads(TESTMODULE_MODULEMD)
mmd.version = 20170219191323
mmd.xmd['mbs']['scmurl'] = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79'
mmd.xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba'
mmd = Modulemd.Module().new_from_string(TESTMODULE_MODULEMD)
mmd.upgrade()
mmd.set_version(20170219191323)
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['scmurl'] = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79'
xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba'
mmd.set_xmd(glib.dict_values(xmd))
pdc_module_reuse.endpoints['unreleasedvariants']['GET'].append(
copy.deepcopy(pdc_module_reuse.endpoints['unreleasedvariants']['GET'][-1]))
pdc_module_reuse.endpoints['unreleasedvariants']['GET'][-1].update({
'variant_uid': 'testmodule:master:{0}'.format(mmd.version),
'variant_release': str(mmd.version),
'variant_uid': 'testmodule:master:{0}'.format(mmd.get_version()),
'variant_release': str(mmd.get_version()),
'modulemd': mmd.dumps(),
'koji_tag': 'module-de3adf79caf3e1b8'
})
mmd.version = 20180205135154
mmd.xmd['mbs']['scmurl'] = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a'
mmd.xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8'
mmd.set_version(20180205135154)
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['scmurl'] = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a'
xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8'
mmd.set_xmd(glib.dict_values(xmd))
pdc_module_reuse.endpoints['unreleasedvariants']['GET'].append(
copy.deepcopy(pdc_module_reuse.endpoints['unreleasedvariants']['GET'][-1]))
pdc_module_reuse.endpoints['unreleasedvariants']['GET'][-1].update({
'variant_uid': 'testmodule:master:{0}'.format(mmd.version),
'variant_release': str(mmd.version),
'variant_uid': 'testmodule:master:{0}'.format(mmd.get_version()),
'variant_release': str(mmd.get_version()),
'modulemd': mmd.dumps(),
'koji_tag': 'module-fe3adf73caf3e1b7',
'rpms': [],

6
tests/staged_data/platform.yaml
View File

@@ -1,6 +1,10 @@
document: modulemd
version: 1
data:
description: Fedora 28 traditional base
name: platform
license:
module: [MIT]
profiles:
buildroot:
rpms: [bash, bzip2, coreutils, cpio, diffutils, fedora-release, findutils, gawk,
@@ -17,5 +21,3 @@ data:
buildrequires: {}
commit: virtual
requires: {}
document: modulemd
version: 1

39
tests/staged_data/testmodule_mse.yaml Normal file
View File

@@ -0,0 +1,39 @@
document: modulemd
version: 2
data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And can be used for
testing the build and release pipeline.
license:
module:
- MIT
dependencies:
- buildrequires:
platform: []
requires:
platform: []
font: [a, b, c]
references:
community: https://docs.pagure.org/modularity/
documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
profiles:
default:
rpms:
- tangerine
api:
rpms:
- perl-Tangerine
- tangerine
components:
rpms:
perl-List-Compare:
rationale: A dependency of tangerine.
ref: master
perl-Tangerine:
rationale: Provides API for this module and is a dependency of tangerine.
ref: master
tangerine:
rationale: Provides API for this module.
ref: master
buildorder: 10

38
tests/staged_data/testmodule_v2.yaml Normal file
View File

@@ -0,0 +1,38 @@
document: modulemd
version: 2
data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And can be used for
testing the build and release pipeline.
license:
module:
- MIT
dependencies:
- buildrequires:
platform: [f28]
requires:
platform: [f28]
references:
community: https://docs.pagure.org/modularity/
documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
profiles:
default:
rpms:
- tangerine
api:
rpms:
- perl-Tangerine
- tangerine
components:
rpms:
perl-List-Compare:
rationale: A dependency of tangerine.
ref: master
perl-Tangerine:
rationale: Provides API for this module and is a dependency of tangerine.
ref: master
tangerine:
rationale: Provides API for this module.
ref: master
buildorder: 10

10
tests/test_build/test_build.py
View File

@@ -37,6 +37,7 @@ from module_build_service import db, models, conf, build_logs
from mock import patch, PropertyMock, Mock
from werkzeug.datastructures import FileStorage
import kobo
import pytest
from tests import app, reuse_component_init_data, clean_database
import json
@@ -317,15 +318,20 @@ class TestBuild:
except Exception:
pass
@pytest.mark.parametrize('mmd_version', [1, 2])
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_build(self, mocked_scm, mocked_get_user, conf_system, dbg,
pdc_module_inactive):
pdc_module_inactive, mmd_version):
"""
Tests the build of testmodule.yaml using FakeModuleBuilder which
succeeds everytime.
"""
FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
if mmd_version == 1:
yaml_file = 'testmodule.yaml'
else:
yaml_file = 'testmodule_v2.yaml'
FakeSCM(mocked_scm, 'testmodule', yaml_file,
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(

95
tests/test_builder/test_mock.py
View File

@@ -5,10 +5,14 @@ import tempfile
import shutil
import kobo.rpmlib
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from module_build_service import conf
from module_build_service.models import ModuleBuild, ComponentBuild, make_session
from module_build_service.builder.MockModuleBuilder import MockModuleBuilder
from module_build_service import glib
from tests import clean_database
@@ -45,56 +49,55 @@ class TestMockModuleBuilder:
]
base_dir = os.path.abspath(os.path.dirname(__file__))
modulemd_path = os.path.join(
base_dir, '..', 'staged_data', 'testmodule-with-filters.yaml')
mmd = Modulemd.Module().new_from_file(os.path.join(
base_dir, '..', 'staged_data', 'testmodule-with-filters.yaml'))
mmd.upgrade()
with open(modulemd_path, "r") as fd:
module = ModuleBuild.create(
session,
conf,
name="mbs-testmodule",
stream="test",
version="20171027111452",
modulemd=fd.read(),
scmurl="file:///testdir",
username="test",
)
module.koji_tag = "module-mbs-testmodule-test-20171027111452"
md = module.mmd()
md.xmd = {
'mbs': {
'rpms': {
'ed': {'ref': '01bf8330812fea798671925cc537f2f29b0bd216'},
'mksh': {'ref': 'f70fd11ddf96bce0e2c64309706c29156b39141d'}
module = ModuleBuild.create(
session,
conf,
name="mbs-testmodule",
stream="test",
version="20171027111452",
modulemd=mmd.dumps(),
scmurl="file:///testdir",
username="test",
)
module.koji_tag = "module-mbs-testmodule-test-20171027111452"
mmd.set_xmd(glib.dict_values({
'mbs': {
'rpms': {
'ed': {'ref': '01bf8330812fea798671925cc537f2f29b0bd216'},
'mksh': {'ref': 'f70fd11ddf96bce0e2c64309706c29156b39141d'}
},
'buildrequires':
{
'host': {
'version': '20171024133034',
'filtered_rpms': [],
'stream': 'master',
'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45'
},
'buildrequires':
{
'host': {
'version': '20171024133034',
'filtered_rpms': [],
'stream': 'master',
'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45'
},
'platform': {
'version': '20171028112959',
'filtered_rpms': [],
'stream': 'master',
'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'}
},
'scmurl': 'file:///testdir',
'commit': '5566bc792ec7a03bb0e28edd1b104a96ba342bd8',
'requires': {
'platform': {
'version': '20171028112959',
'filtered_rpms': [],
'stream': 'master',
'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'}
}
'platform': {
'version': '20171028112959',
'filtered_rpms': [],
'stream': 'master',
'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'}
},
'scmurl': 'file:///testdir',
'commit': '5566bc792ec7a03bb0e28edd1b104a96ba342bd8',
'requires': {
'platform': {
'version': '20171028112959',
'filtered_rpms': [],
'stream': 'master',
'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'}
}
}
module.modulemd = md.dumps()
module.batch = batch
session.add(module)
}))
module.modulemd = mmd.dumps()
module.batch = batch
session.add(module)
for build in comp_builds:
cb = ComponentBuild(**dict(build, format="rpms", state=state))

17
tests/test_models/__init__.py
View File

@@ -21,11 +21,13 @@
# Written by Ralph Bean <rbean@redhat.com>
import os
from datetime import datetime
import module_build_service
import modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from datetime import datetime
from tests import db, clean_database
from module_build_service.config import init_config
from module_build_service.models import ModuleBuild, BUILD_STATES
@@ -38,13 +40,12 @@ datadir = os.path.dirname(__file__) + '/data/'
def module_build_from_modulemd(yaml):
mmd = modulemd.ModuleMetadata()
mmd.loads(yaml)
mmd = Modulemd.Module().new_from_string(yaml)
mmd.upgrade()
build = ModuleBuild()
build.name = mmd.name
build.stream = mmd.stream
build.version = mmd.version
build.name = mmd.get_name()
build.stream = mmd.get_stream()
build.version = mmd.get_version()
build.state = BUILD_STATES['ready']
build.modulemd = yaml
build.koji_tag = None

8
tests/test_models/test_models.py
View File

@@ -22,7 +22,9 @@
import os
import modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from tests.test_models import init_data
from module_build_service import conf
@@ -63,10 +65,10 @@ class TestModels:
""" Test that the build_context, runtime_context, and context hashes are correctly
determined"""
build = ModuleBuild.query.filter_by(id=1).one()
mmd = modulemd.ModuleMetadata()
yaml_path = os.path.join(
os.path.dirname(__file__), '..', 'staged_data', 'testmodule_dependencies.yaml')
mmd.load(yaml_path)
mmd = Modulemd.Module().new_from_file(yaml_path)
mmd.upgrade()
build.modulemd = mmd.dumps()
build.build_context, build.runtime_context = ModuleBuild.contexts_from_mmd(build.modulemd)
assert build.build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7'

42
tests/test_resolver/test_pdc.py
View File

@@ -25,14 +25,16 @@ import copy
from mock import patch, PropertyMock
import pytest
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
import module_build_service.resolver as mbs_resolver
import module_build_service.utils
import module_build_service.models
from module_build_service import app, db
from module_build_service import glib
import tests
import modulemd
base_dir = os.path.join(os.path.dirname(__file__), "..")
@@ -76,10 +78,12 @@ class TestPDCModule:
if empty_buildrequires:
expected = set()
pdc_item = pdc_module_active.endpoints['unreleasedvariants']['GET'][-1]
mmd = modulemd.ModuleMetadata()
mmd.loads(pdc_item['modulemd'])
mmd.buildrequires = {}
mmd.xmd['mbs']['buildrequires'] = {}
mmd = Modulemd.Module().new_from_string(pdc_item['modulemd'])
# Wipe out the dependencies
mmd.set_dependencies()
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['buildrequires'] = {}
mmd.set_xmd(glib.dict_values(xmd))
pdc_item.update({
'modulemd': mmd.dumps(),
'build_deps': []
@@ -101,21 +105,25 @@ class TestPDCModule:
pdc_module_active.endpoints['unreleasedvariants']['GET'].append(
copy.deepcopy(pdc_module_active.endpoints['unreleasedvariants']['GET'][-1]))
pdc_item = pdc_module_active.endpoints['unreleasedvariants']['GET'][-1]
mmd = modulemd.ModuleMetadata()
mmd.loads(pdc_item['modulemd'])
mmd.name = 'testmodule2'
mmd.version = 20180123171545
mmd.requires['testmodule'] = 'master'
mmd.xmd['mbs']['requires']['testmodule'] = {
mmd = Modulemd.Module().new_from_string(pdc_item['modulemd'])
mmd.set_name('testmodule2')
mmd.set_version(20180123171545)
requires = mmd.get_dependencies()[0].get_requires()
requires['testmodule'] = Modulemd.SimpleSet()
requires['testmodule'].add('master')
mmd.get_dependencies()[0].set_requires(requires)
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['requires']['testmodule'] = {
'filtered_rpms': [],
'ref': '620ec77321b2ea7b0d67d82992dda3e1d67055b4',
'stream': 'master',
'version': '20180205135154'
}
mmd.set_xmd(glib.dict_values(xmd))
pdc_item.update({
'variant_id': 'testmodule2',
'variant_name': 'testmodule2',
'variant_release': str(mmd.version),
'variant_release': str(mmd.get_version()),
'koji_tag': 'module-ae2adf69caf0e1b6',
'modulemd': mmd.dumps()
})
@@ -164,8 +172,8 @@ class TestPDCModule:
def test_resolve_profiles(self, pdc_module_active):
yaml_path = os.path.join(
base_dir, 'staged_data', 'formatted_testmodule.yaml')
mmd = modulemd.ModuleMetadata()
mmd.load(yaml_path)
mmd = Modulemd.Module().new_from_file(yaml_path)
mmd.upgrade()
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
result = resolver.resolve_profiles(mmd, ('buildroot', 'srpm-buildroot'))
expected = {
@@ -193,8 +201,8 @@ class TestPDCModule:
yaml_path = os.path.join(
base_dir, 'staged_data', 'formatted_testmodule.yaml')
mmd = modulemd.ModuleMetadata()
mmd.load(yaml_path)
mmd = Modulemd.Module().new_from_file(yaml_path)
mmd.upgrade()
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='pdc')
result = resolver.resolve_profiles(mmd, ('buildroot', 'srpm-buildroot'))
expected = {

5
tests/test_scheduler/test_module_init.py
View File

@@ -22,6 +22,7 @@
import os
from mock import patch, PropertyMock
from gi.repository import GLib
from tests import conf, clean_database
from tests.test_views.test_views import FakeSCM
@@ -66,7 +67,7 @@ class TestModuleInit:
# Make sure the module entered the wait state
assert build.state == 1, build.state
# Make sure format_mmd was run properly
assert type(build.mmd().xmd['mbs']) is dict
assert type(build.mmd().get_xmd()['mbs']) is GLib.Variant
@patch('module_build_service.scm.SCM')
def test_init_scm_not_available(self, mocked_scm, pdc):
@@ -120,7 +121,7 @@ class TestModuleInit:
'foo': {'ref': '93dea37599'},
'file': {'ref': 'a2740663f8'},
}
assert build.mmd().xmd['mbs']['rpms'] == xmd_rpms
assert build.mmd().get_xmd()['mbs']['rpms'] == xmd_rpms
@patch('module_build_service.models.ModuleBuild.from_module_event')
@patch('module_build_service.scm.SCM')

22
tests/test_scheduler/test_module_wait.py
View File

@@ -24,7 +24,9 @@ import mock
from mock import patch
import module_build_service.messaging
import module_build_service.scheduler.handlers.modules
import modulemd as _modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
import os
import koji
from tests import conf, db, app, scheduler_init_data
@@ -66,12 +68,10 @@ class TestModuleWait:
}
mocked_module_build.extended_json = mocked_module_build.json
mmd = _modulemd.ModuleMetadata()
formatted_testmodule_yml_path = os.path.join(
base_dir, 'staged_data', 'formatted_testmodule.yaml')
with open(formatted_testmodule_yml_path, 'r') as f:
mmd.loads(f)
mmd = Modulemd.Module().new_from_file(formatted_testmodule_yml_path)
mmd.upgrade()
mocked_module_build.id = 1
mocked_module_build.mmd.return_value = mmd
mocked_module_build.component_builds = []
@@ -172,9 +172,9 @@ class TestModuleWait:
Test that build.cg_build_koji_tag fallbacks to default tag.
"""
with app.app_context():
base_mmd = _modulemd.ModuleMetadata()
base_mmd.name = "base-runtime"
base_mmd.stream = "f27"
base_mmd = Modulemd.Module()
base_mmd.set_name("base-runtime")
base_mmd.set_stream("f27")
scheduler_init_data()
koji_session = mock.MagicMock()
@@ -217,9 +217,9 @@ class TestModuleWait:
Test that build.cg_build_koji_tag is set.
"""
with app.app_context():
base_mmd = _modulemd.ModuleMetadata()
base_mmd.name = "base-runtime"
base_mmd.stream = "f27"
base_mmd = Modulemd.Module()
base_mmd.set_name("base-runtime")
base_mmd.set_stream("f27")
scheduler_init_data()
koji_session = mock.MagicMock()

68
tests/test_utils/test_utils.py
View File

@@ -22,7 +22,9 @@ import tempfile
from os import path, mkdir
from shutil import copyfile, rmtree
from datetime import datetime
import modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
from werkzeug.datastructures import FileStorage
from mock import patch
import module_build_service.utils
@@ -37,6 +39,7 @@ import pytest
import module_build_service.scheduler.handlers.components
from module_build_service.builder.base import GenericBuilder
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
from module_build_service import glib
from tests import app
BASE_DIR = path.abspath(path.dirname(__file__))
@@ -90,7 +93,8 @@ class TestUtilsComponentReuse:
second_module_build = models.ModuleBuild.query.filter_by(id=2).one()
if changed_component:
mmd = second_module_build.mmd()
mmd.components.rpms[changed_component].ref = '00ea1da4192a2030f9ae023de3b3143ed647bbab'
mmd.get_rpm_components()['tangerine'].set_ref(
'00ea1da4192a2030f9ae023de3b3143ed647bbab')
second_module_build.modulemd = mmd.dumps()
second_module_changed_component = models.ComponentBuild.query.filter_by(
package=changed_component, module_id=2).one()
@@ -130,7 +134,7 @@ class TestUtilsComponentReuse:
def test_get_reusable_component_different_rpm_macros(self):
second_module_build = models.ModuleBuild.query.filter_by(id=2).one()
mmd = second_module_build.mmd()
mmd.buildopts.rpms.macros = "%my_macro 1"
mmd.set_rpm_buildopts({'macros': '%my_macro 1'})
second_module_build.modulemd = mmd.dumps()
db.session.commit()
@@ -145,8 +149,10 @@ class TestUtilsComponentReuse:
def test_get_reusable_component_different_buildrequires_hash(self):
second_module_build = models.ModuleBuild.query.filter_by(id=2).one()
mmd = second_module_build.mmd()
mmd.xmd['mbs']['buildrequires']['platform']['ref'] = \
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['buildrequires']['platform']['ref'] = \
'da39a3ee5e6b4b0d3255bfef95601890afd80709'
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
db.session.commit()
@@ -166,14 +172,18 @@ class TestUtilsComponentReuse:
def test_get_reusable_component_different_buildrequires(self):
second_module_build = models.ModuleBuild.query.filter_by(id=2).one()
mmd = second_module_build.mmd()
mmd.buildrequires = {'some_module': 'master'}
mmd.xmd['mbs']['buildrequires'] = {
br_list = Modulemd.SimpleSet()
br_list.add('master')
mmd.get_dependencies()[0].set_buildrequires({'some_module': br_list})
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['buildrequires'] = {
'some_module': {
'ref': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
'stream': 'master',
'version': '20170123140147'
}
}
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
db.session.commit()
@@ -218,8 +228,8 @@ class TestUtilsComponentReuse:
mock_submit_args = mock_submit.call_args[0]
username_arg = mock_submit_args[0]
mmd_arg = mock_submit_args[2]
assert mmd_arg.stream == stream
assert "\n\n%__spec_check_pre exit 0\n" in mmd_arg.buildopts.rpms.macros
assert mmd_arg.get_stream() == stream
assert "\n\n%__spec_check_pre exit 0\n" in mmd_arg.get_rpm_buildopts()['macros']
assert username_arg == username
rmtree(module_dir)
@@ -254,21 +264,23 @@ class TestUtils:
return hashes_returned[ref]
mocked_scm.return_value.get_latest = mocked_get_latest
mmd = modulemd.ModuleMetadata()
mmd.load(path.join(BASE_DIR, '..', 'staged_data', 'testmodule.yaml'))
mmd = Modulemd.Module().new_from_file(
path.join(BASE_DIR, '..', 'staged_data', 'testmodule.yaml'))
mmd.upgrade()
# Modify the component branches so we can identify them later on
mmd.components.rpms['perl-Tangerine'].ref = 'f28'
mmd.components.rpms['tangerine'].ref = 'f27'
mmd.get_rpm_components()['perl-Tangerine'].set_ref('f28')
mmd.get_rpm_components()['tangerine'].set_ref('f27')
module_build_service.utils.format_mmd(mmd, scmurl)
# Make sure that original refs are not changed.
mmd_pkg_refs = [pkg.ref for pkg in mmd.components.rpms.values()]
mmd_pkg_refs = [pkg.get_ref() for pkg in mmd.get_rpm_components().values()]
assert set(mmd_pkg_refs) == set(hashes_returned.keys())
assert mmd.buildrequires == {'platform': 'f28'}
br = mmd.get_dependencies()[0].get_buildrequires()
assert br.keys() == ['platform']
assert br.values()[0].get() == ['f28']
xmd = {
'mbs': {
'commit': None,
'commit': '',
'buildrequires': {
'platform': {
'ref': 'virtual',
@@ -285,14 +297,14 @@ class TestUtils:
'perl-List-Compare': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'},
'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'},
'tangerine': {'ref': '5deef23acd2367d8b8d5a621a2fc568b695bc3bd'}},
'scmurl': None
'scmurl': ''
}
}
if scmurl:
xmd['mbs']['commit'] = '620ec77321b2ea7b0d67d82992dda3e1d67055b4'
xmd['mbs']['scmurl'] = scmurl
assert mmd.xmd == xmd
mmd_xmd = glib.from_variant_dict(mmd.get_xmd())
assert mmd_xmd == xmd
def test_get_reusable_component_shared_userspace_ordering(self):
"""
@@ -423,9 +435,9 @@ class TestUtils:
'fbed359411a1baa08d4a88e0d12d426fbf8f602c']
testmodule_mmd_path = path.join(BASE_DIR, '..', 'staged_data', 'testmodule.yaml')
mmd = modulemd.ModuleMetadata()
mmd.load(testmodule_mmd_path)
mmd.name = 'testmodule-variant'
mmd = Modulemd.Module().new_from_file(testmodule_mmd_path)
mmd.upgrade()
mmd.set_name('testmodule-variant')
module_build = module_build_service.models.ModuleBuild()
module_build.name = 'testmodule-variant'
module_build.stream = 'master'
@@ -441,11 +453,13 @@ class TestUtils:
db.session.add(module_build)
db.session.commit()
# Rename the the modulemd to include
mmd.name = 'testmodule'
mmd.set_name('testmodule')
# Remove perl-Tangerine and tangerine from the modulemd to include so only one
# component conflicts
mmd.components.rpms.pop('perl-Tangerine')
mmd.components.rpms.pop('tangerine')
comps = mmd.get_rpm_components()
del comps['perl-Tangerine']
del comps['tangerine']
mmd.set_rpm_components(comps)
error_msg = (
'The included module "testmodule" in "testmodule-variant" have '
@@ -469,8 +483,8 @@ class TestUtils:
testmodule_mmd_path = path.join(
BASE_DIR, '..', 'staged_data', 'testmodule.yaml')
mmd = modulemd.ModuleMetadata()
mmd.load(testmodule_mmd_path)
mmd = Modulemd.Module().new_from_file(testmodule_mmd_path)
mmd.upgrade()
module_build = module_build_service.models.ModuleBuild()
module_build.name = 'testmodule'
module_build.stream = 'master'

24
tests/test_views/test_views.py
View File

@@ -22,7 +22,9 @@
import json
import modulemd as _modulemd
import gi
gi.require_version('Modulemd', '1.0') # noqa
from gi.repository import Modulemd
import module_build_service.scm
from mock import patch, PropertyMock
@@ -533,8 +535,8 @@ class TestViews:
assert len(data['state_trace']) == 1
assert data['state_trace'][0]['state'] == 0
assert data['tasks'] == {}
mmd = _modulemd.ModuleMetadata()
mmd.loads(data["modulemd"])
mmd = Modulemd.Module().new_from_string(data['modulemd'])
mmd.upgrade()
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
@@ -661,7 +663,7 @@ class TestViews:
{'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
assert data['message'].startswith('Invalid modulemd:') is True
assert data['message'].startswith('The following invalid modulemd was encountered') is True
assert data['status'] == 422
assert data['error'] == 'Unprocessable Entity'
@@ -806,6 +808,20 @@ class TestViews:
'does not match the branch "master"')
assert data['error'] == 'Bad Request'
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_build_mse_unsupported(self, mocked_scm, mocked_get_user):
FakeSCM(mocked_scm, 'testmodule', 'testmodule_mse.yaml',
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
assert data['status'] == 422
assert data['message'] == 'Module stream expansion is not yet supported in MBS'
assert data['error'] == 'Unprocessable Entity'
@patch('module_build_service.auth.get_user', return_value=user)
def test_submit_build_set_owner(self, mocked_get_user):
data = {