modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-08 21:29:17 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #595 Fix PEP8 issues

Browse Source
This commit is contained in:
Ralph Bean
2017-07-18 13:26:38 +00:00
parent 9b19e52bc7 04545b449d
commit aebfb1b812
43 changed files with 282 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bootstrap/fix-json.py
View File

@@ -1,17 +1,12 @@
import json
import requests
import os
import sys
import yaml
import pdc_client
filename = 'bootstrap-master-1.json'
print("Reading %s" % filename)
with open(filename, 'r') as f:
entry = json.loads(f.read())
print entry['modulemd']
print(entry['modulemd'])
modulemd = yaml.load(entry['modulemd'])
mbs = {}
mbs['commit'] = modulemd['data']['xmd']['mbs_commit']

1
bootstrap/import-to-pdc.py
View File

@@ -1,5 +1,4 @@
import json
import requests
import os
import sys

5
conf/config.py
View File

@@ -5,7 +5,7 @@ from os import path
confdir = path.abspath(path.dirname(__file__))
# use parent dir as dbdir else fallback to current dir
dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \
else confdir
else confdir
class BaseConfiguration(object):
@@ -67,7 +67,7 @@ class BaseConfiguration(object):
ALLOWED_GROUPS = set([
'packager',
#'modularity-wg',
# 'modularity-wg',
])
# Available backends are: console, file, journal.
@@ -172,6 +172,7 @@ class TestConfiguration(BaseConfiguration):
class ProdConfiguration(BaseConfiguration):
pass
class LocalBuildConfiguration(BaseConfiguration):
LOG_LEVEL = 'debug'
MESSAGING = 'in_memory'

2
fedmsg.d/module_build_service.py
View File

@@ -13,7 +13,7 @@ config = {
"endpoints": {
"fedora-infrastructure": [
# Just listen to staging for now, not to production (spam!)
#"tcp://hub.fedoraproject.org:9940",
# "tcp://hub.fedoraproject.org:9940",
"tcp://stg.fedoraproject.org:9940",
],
},

7
module_build_service/__init__.py
View File

@@ -59,6 +59,7 @@ conf = init_config(app)
db = SQLAlchemy(app)
@app.errorhandler(ValidationError)
def validationerror_error(e):
"""Flask error handler for ValidationError exceptions"""
@@ -104,6 +105,7 @@ init_logging(conf)
log = getLogger(__name__)
build_logs = ModuleBuildLogs(conf.build_logs_dir)
def get_url_for(*args, **kwargs):
"""
flask.url_for wrapper which creates the app_context on-the-fly.
@@ -116,8 +118,9 @@ def get_url_for(*args, **kwargs):
app.config['SERVER_NAME'] = 'localhost'
with app.app_context():
log.debug("WARNING: get_url_for() has been called without the Flask "
"app_context. That can lead to SQLAlchemy errors caused by "
"multiple session being used in the same time.")
"app_context. That can lead to SQLAlchemy errors caused by "
"multiple session being used in the same time.")
return url_for(*args, **kwargs)
from module_build_service import views

18
module_build_service/auth.py
View File

@@ -37,18 +37,20 @@ def _json_loads(content):
client_secrets = None
def _load_secrets():
global client_secrets
if client_secrets:
return
if not "OIDC_CLIENT_SECRETS" in app.config:
if "OIDC_CLIENT_SECRETS" not in app.config:
raise Forbidden("OIDC_CLIENT_SECRETS must be set in server config.")
secrets = _json_loads(open(app.config['OIDC_CLIENT_SECRETS'],
'r').read())
'r').read())
client_secrets = list(secrets.values())[0]
def _get_token_info(token):
"""
Asks the token_introspection_uri for the validity of a token.
@@ -57,9 +59,9 @@ def _get_token_info(token):
return None
request = {'token': token,
'token_type_hint': 'Bearer',
'client_id': client_secrets['client_id'],
'client_secret': client_secrets['client_secret']}
'token_type_hint': 'Bearer',
'client_id': client_secrets['client_id'],
'client_secret': client_secrets['client_secret']}
headers = {'Content-type': 'application/x-www-form-urlencoded'}
resp = requests.post(client_secrets['token_introspection_uri'], data=request, headers=headers)
@@ -89,7 +91,7 @@ def get_user(request):
_load_secrets()
if not "authorization" in request.headers:
if "authorization" not in request.headers:
raise Unauthorized("No 'authorization' header found.")
header = request.headers['authorization'].strip()
@@ -105,10 +107,10 @@ def get_user(request):
log.exception(error)
raise Exception(error)
if not data or not "active" in data or not data["active"]:
if not data or "active" not in data or not data["active"]:
raise Unauthorized("OIDC token invalid or expired.")
if not "OIDC_REQUIRED_SCOPE" in app.config:
if "OIDC_REQUIRED_SCOPE" not in app.config:
raise Forbidden("OIDC_REQUIRED_SCOPE must be set in server config.")
presented_scopes = data['scope'].split(' ')

2
module_build_service/builder/CoprModuleBuilder.py
View File

@@ -35,7 +35,6 @@ from copr.client import CoprClient
from copr.exceptions import CoprRequestException
from module_build_service import log
from module_build_service.models import ModuleBuild
import module_build_service.scm
import module_build_service.utils
@@ -70,7 +69,6 @@ class CoprModuleBuilder(GenericBuilder):
self.chroot = "custom-1-x86_64"
self.__prep = False
@classmethod
def _get_client(cls, config):
return CoprClient.create_from_file_config(config.copr_config)

10
module_build_service/builder/KojiContentGenerator.py
View File

@@ -57,7 +57,6 @@ class KojiContentGenerator(object):
self.mmd = module.modulemd
self.config = config
def __repr__(self):
return "<KojiContentGenerator module: %s>" % (self.module_name)
@@ -185,7 +184,7 @@ class KojiContentGenerator(object):
try:
rpms, builds = session.listTaggedRPMS(tag, latest=True)
except koji.GenericError as e:
except koji.GenericError:
log.exception("Failed to list rpms in tag %r", tag)
# If the tag doesn't exist.. then there are no rpms in that tag.
return []
@@ -244,7 +243,6 @@ class KojiContentGenerator(object):
}
return ret
def _get_output(self, output_path):
ret = []
rpms = self._koji_rpms_in_tag(self.module.koji_tag)
@@ -302,7 +300,6 @@ class KojiContentGenerator(object):
return ret
def _get_content_generator_metadata(self, output_path):
ret = {
u"metadata_version": 0,
@@ -327,11 +324,10 @@ class KojiContentGenerator(object):
log_path = os.path.join(prepdir, "build.log")
try:
shutil.copy(build_logs.path(self.module.id), log_path)
except IOError, e:
except IOError as e:
log.exception(e)
return prepdir
def koji_import(self):
"""This method imports given module into the configured koji instance as
a content generator based build
@@ -345,7 +341,7 @@ class KojiContentGenerator(object):
build_info = session.CGImport(metadata, file_dir)
log.debug("Content generator import done: %s",
json.dumps(build_info, sort_keys=True, indent=4))
except Exception, e:
except Exception as e:
log.exception("Content generator import failed: %s", e)
raise e
finally:

48
module_build_service/builder/KojiModuleBuilder.py
View File

@@ -40,7 +40,7 @@ import threading
import munch
from OpenSSL.SSL import SysCallError
from module_build_service import conf, log, db
from module_build_service import log
import module_build_service.scm
import module_build_service.utils
from module_build_service.builder.utils import execute_cmd
@@ -80,9 +80,9 @@ class KojiModuleBuilder(GenericBuilder):
raise ValueError("No koji_arches specified in the config.")
# These eventually get populated by calling _connect and __prep is set to True
self.module_tag = None # string
self.module_build_tag = None # string
self.module_target = None # A koji target dict
self.module_tag = None # string
self.module_build_tag = None # string
self.module_target = None # A koji target dict
self.build_priority = config.koji_build_priority
self.components = components
@@ -127,11 +127,10 @@ class KojiModuleBuilder(GenericBuilder):
log.info("%r buildroot is not yet ready.. wait." % self)
return ready
@staticmethod
def get_disttag_srpm(disttag, module_build):
#Taken from Karsten's create-distmacro-pkg.sh
# Taken from Karsten's create-distmacro-pkg.sh
# - however removed any provides to system-release/redhat-release
name = 'module-build-macros'
@@ -284,7 +283,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
# Create or update individual tags
self.module_tag = self._koji_create_tag(
self.tag_name, self.arches, perm="admin") # the main tag needs arches so pungi can dump it
self.tag_name, self.arches, perm="admin") # the main tag needs arches so pungi can dump it
self.module_build_tag = self._koji_create_tag(
self.tag_name + "-build", self.arches, perm="admin")
@@ -370,7 +369,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
log.info("Waiting for task_id=%s to finish" % task_id)
timeout = 60 * 60 # 60 minutes
timeout = 60 * 60 # 60 minutes
@module_build_service.utils.retry(timeout=timeout, wait_on=koji.GenericError)
def get_result():
log.debug("Waiting for task_id=%s to finish" % task_id)
@@ -467,9 +467,9 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
source, task_info['task_id'], self))
return task_info['task_id'], koji.BUILD_STATES['COMPLETE'], 'Build already exists.', task_info['nvr']
self._koji_whitelist_packages([artifact_name,])
self._koji_whitelist_packages([artifact_name])
if '://' not in source:
#treat source as an srpm and upload it
# treat source as an srpm and upload it
serverdir = _unique_path('cli-build')
callback = None
self.koji_session.uploadWrapper(source, serverdir, callback=callback)
@@ -480,8 +480,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
# The reason is that it is faster to build this RPM in
# already existing shared target, because Koji does not need to do
# repo-regen.
if (artifact_name == "module-build-macros"
and self.config.koji_build_macros_target):
if (artifact_name == "module-build-macros" and
self.config.koji_build_macros_target):
module_target = self.config.koji_build_macros_target
else:
module_target = self.module_target['name']
@@ -536,13 +536,14 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
priority = 0
if inheritance_data:
priority = inheritance_data[-1]['priority'] + 10
def record_exists(parent_id, data):
for item in data:
if parent_id == item['parent_id']:
return True
return False
for parent in parent_tags: # We expect that they're sorted
for parent in parent_tags: # We expect that they're sorted
parent = self._get_tag(parent)
if record_exists(parent['id'], inheritance_data):
continue
@@ -571,10 +572,10 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
raise ValueError("Expected dict {'group' : [str(package1), ...]")
dest_tag = self._get_tag(dest_tag)['name']
existing_groups = dict([
(p['name'], p['group_id'])
for p in self.koji_session.getTagGroups(dest_tag, inherit=False)
])
existing_groups = dict([(p['name'], p['group_id'])
for p
in self.koji_session.getTagGroups(dest_tag, inherit=False)
])
for group, packages in groups.items():
group_id = existing_groups.get(group, None)
@@ -589,7 +590,6 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
for pkg in packages:
self.koji_session.groupPackageListAdd(dest_tag, group, pkg)
@module_build_service.utils.validate_koji_tag('tag_name')
def _koji_create_tag(self, tag_name, arches=None, perm=None):
"""
@@ -613,8 +613,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
raise ValueError("Expected list or None on input got %s" % type(arches))
current_arches = []
if taginfo['arches']: # None if none
current_arches = taginfo['arches'].split() # string separated by empty spaces
if taginfo['arches']: # None if none
current_arches = taginfo['arches'].split() # string separated by empty spaces
if set(arches) != set(current_arches):
opts['arches'] = " ".join(arches)
@@ -629,7 +629,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
raise ValueError("Unknown permissions %s" % perm)
perm_id = perm_ids[perm]
if taginfo['perm'] not in (perm_id, perm): # check either id or the string
if taginfo['perm'] not in (perm_id, perm): # check either id or the string
opts['perm'] = perm_id
opts['extra'] = {
@@ -638,7 +638,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
# edit tag with opts
self.koji_session.editTag2(tag_name, **opts)
return self._get_tag(tag_name) # Return up2date taginfo
return self._get_tag(tag_name) # Return up2date taginfo
def _koji_whitelist_packages(self, packages, tags=None):
if not tags:
@@ -681,7 +681,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
if not target_info:
target_info = self.koji_session.createBuildTarget(name, build_tag['name'], dest_tag['name'])
else: # verify whether build and destination tag matches
else: # verify whether build and destination tag matches
if build_tag['name'] != target_info['build_tag_name']:
raise SystemError("Target references unexpected build_tag_name. Got '%s', expected '%s'. Please contact administrator." % (target_info['build_tag_name'], build_tag['name']))
if dest_tag['name'] != target_info['dest_tag_name']:
@@ -725,7 +725,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
# TODO: https://pagure.io/fm-orchestrator/issue/397
# Subj: Do not mix target/tag when looking for component builds
if (c.package == task_opts['mbs_artifact_name'] and
c.module_build.koji_tag == task_opts['mbs_module_target']):
c.module_build.koji_tag == task_opts['mbs_module_target']):
tasks.append(task)
return tasks

11
module_build_service/builder/MockModuleBuilder.py
View File

@@ -28,10 +28,9 @@ import koji
import kobo.rpmlib
import modulemd
import pipes
import yaml
import threading
from module_build_service import conf, log, db
from module_build_service import conf, log
import module_build_service.scm
import module_build_service.utils
import module_build_service.scheduler
@@ -104,7 +103,7 @@ mdpolicy=group:primary
self.tag_name = tag_name
self.config = config
self.groups = []
self.arch = "x86_64" # TODO: We may need to change that in the future
self.arch = "x86_64" # TODO: We may need to change that in the future
self.yum_conf = MockModuleBuilder.MOCK_YUM_CONF_TEMPLATE
# Create main directory for this tag
@@ -183,7 +182,7 @@ mdpolicy=group:primary
pkglist_f.close()
mmd_path = os.path.join(path, "modules.yaml")
modulemd.dump_all(mmd_path, [ m1_mmd ])
modulemd.dump_all(mmd_path, [m1_mmd])
# Generate repo and inject modules.yaml there.
execute_cmd(['/usr/bin/createrepo_c', '--pkglist', pkglist, path])
@@ -477,8 +476,8 @@ class SCMBuilder(BaseBuilder):
# this code.
distgit_get_branch = \
"sh -c {}'; git -C {} checkout {}'".format(pipes.quote(distgit_get),
artifact_name,
branch)
artifact_name,
branch)
f.writelines([
"config_opts['scm'] = True\n",

2
module_build_service/builder/__init__.py
View File

@@ -1,6 +1,7 @@
from module_build_service import conf
from base import GenericBuilder
from KojiModuleBuilder import KojiModuleBuilder
from MockModuleBuilder import MockModuleBuilder
__all__ = [
GenericBuilder
@@ -9,7 +10,6 @@ __all__ = [
GenericBuilder.register_backend_class(KojiModuleBuilder)
from MockModuleBuilder import MockModuleBuilder
GenericBuilder.register_backend_class(MockModuleBuilder)
if conf.system == "copr":

2
module_build_service/builder/base.py
View File

@@ -60,6 +60,8 @@ Koji workflow
8) (optional) wait for selected builds to be available in buildroot
"""
class GenericBuilder(six.with_metaclass(ABCMeta)):
"""
External Api for builders

4
module_build_service/builder/utils.py
View File

@@ -8,7 +8,7 @@ import logging
import urlgrabber.grabber as grabber
import urlgrabber.progress as progress
import module_build_service
from module_build_service import log, scm, messaging
from module_build_service import log
logging.basicConfig(level=logging.DEBUG)
@@ -75,7 +75,7 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
# Get the list of all RPMs and builds in a tag.
try:
rpms, builds = session.listTaggedRPMS(tag, latest=True)
except koji.GenericError as e:
except koji.GenericError:
log.exception("Failed to list rpms in tag %r" % tag)
# Reformat builds so they are dict with build_id as a key.

7
module_build_service/config.py
View File

@@ -109,6 +109,7 @@ def init_config(app):
app.config.from_object(config_section_obj)
return conf
class Path:
"""
Config type for paths. Expands the users home directory.
@@ -189,14 +190,14 @@ class Config(object):
'desc': 'List of allowed koji tag prefixes.'},
'koji_target_delete_time': {
'type': int,
'default': 24*3600,
'default': 24 * 3600,
'desc': 'Time in seconds after which the Koji target of '
'built module is deleted'},
'built module is deleted'},
'koji_enable_content_generator': {
'type': bool,
'default': True,
'desc': 'Enable or disable imports to koji using content '
'generator api'},
'generator api'},
'allow_custom_scmurls': {
'type': bool,
'default': False,

5
module_build_service/logger.py
View File

@@ -53,6 +53,7 @@ levels["info"] = logging.INFO
log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s'
class ModuleBuildFileHandler(logging.FileHandler):
"""
FileHandler subclass which handles only messages generated during
@@ -138,14 +139,16 @@ def str_to_log_level(level):
Available levels are: debug, info, warning, error
"""
if not level in levels:
if level not in levels:
return logging.NOTSET
return levels[level]
def supported_log_backends():
return ("console", "journal", "file")
def init_logging(conf):
"""
Initializes logging according to configuration file.

11
module_build_service/manage.py
View File

@@ -30,20 +30,11 @@ import ssl
from shutil import rmtree
import getpass
import fedmsg.config
import moksha.hub
import moksha.hub.hub
import moksha.hub.reactor
from module_build_service import app, conf, db
from module_build_service import models
from module_build_service.pdc import (
get_pdc_client_session, get_module,
get_module_tag, get_module_build_dependencies)
from module_build_service.utils import (
submit_module_build_from_scm,
)
from module_build_service.messaging import MBSModule
import module_build_service.messaging
import module_build_service.scheduler.consumer
@@ -154,7 +145,7 @@ def build_module_locally(url, branch, skiptests=False):
username = getpass.getuser()
submit_module_build_from_scm(username, url, branch, allow_local_url=True,
skiptests=skiptests)
skiptests=skiptests)
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)

2
module_build_service/messaging.py
View File

@@ -265,6 +265,7 @@ class KojiBuildChange(BaseMessage):
self.module_build_id = module_build_id
self.state_reason = state_reason
class KojiTagChange(BaseMessage):
"""
A class that inherits from BaseMessage to provide a message
@@ -277,6 +278,7 @@ class KojiTagChange(BaseMessage):
self.tag = tag
self.artifact = artifact
class KojiRepoChange(BaseMessage):
""" A class that inherits from BaseMessage to provide a message
object for a repo's info (in fedmsg this replaces the msg dictionary)

8
module_build_service/models.py
View File

@@ -67,6 +67,7 @@ BUILD_STATES = {
INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()}
@contextlib.contextmanager
def _dummy_context_mgr():
"""
@@ -75,6 +76,7 @@ def _dummy_context_mgr():
"""
yield None
@contextlib.contextmanager
def make_session(conf):
"""
@@ -142,7 +144,7 @@ class ModuleBuild(MBSBase):
if not self.batch:
raise ValueError("No batch is in progress: %r" % self.batch)
if state != None:
if state is not None:
return [
component for component in self.component_builds
if component.batch == self.batch and component.state == state
@@ -162,7 +164,7 @@ class ModuleBuild(MBSBase):
if not self.batch:
raise ValueError("No batch is in progress: %r" % self.batch)
if state != None:
if state is not None:
return [
component for component in self.component_builds
if component.batch <= self.batch and component.state == state
@@ -339,7 +341,7 @@ class ModuleBuild(MBSBase):
'state_reason': self.state_reason,
"owner": self.owner,
"name": self.name,
"scmurl": self.scmurl,
"scmurl": self.scmurl,
"time_submitted": self._utc_datetime_to_iso(self.time_submitted),
"time_modified": self._utc_datetime_to_iso(self.time_modified),
"time_completed": self._utc_datetime_to_iso(self.time_completed),

22
module_build_service/pdc.py
View File

@@ -28,13 +28,11 @@
import modulemd
from pdc_client import PDCClient
import copy
import inspect
import pprint
import logging
log = logging.getLogger()
import six
log = logging.getLogger()
def get_pdc_client_session(config):
@@ -57,6 +55,7 @@ def get_pdc_client_session(config):
insecure=config.pdc_insecure,
)
def get_variant_dict(data):
"""
:param data: one of following
@@ -117,7 +116,6 @@ def get_variant_dict(data):
if 'variant_type' not in result.keys():
result['variant_type'] = 'module'
elif is_module_dict(data):
result = {'variant_id': data['name'], 'variant_version': data['version']}
@@ -144,13 +142,14 @@ def variant_dict_from_str(module_str):
release_start = module_str.rfind('-')
version_start = module_str.rfind('-', 0, release_start)
module_info['variant_release'] = module_str[release_start+1:]
module_info['variant_version'] = module_str[version_start+1:release_start]
module_info['variant_release'] = module_str[release_start + 1:]
module_info['variant_version'] = module_str[version_start + 1:release_start]
module_info['variant_id'] = module_str[:version_start]
module_info['variant_type'] = 'module'
return module_info
def get_module(session, module_info, strict=False):
"""
:param session : PDCClient instance
@@ -173,7 +172,7 @@ def get_module(session, module_info, strict=False):
if module_info.get('active'):
query['active'] = module_info['active']
retval = session['unreleasedvariants/'](page_size=-1, **query) # ordering=variant_release...
retval = session['unreleasedvariants/'](page_size=-1, **query) # ordering=variant_release...
# Error handling
if not retval:
@@ -197,6 +196,7 @@ def get_module(session, module_info, strict=False):
return module
def get_module_tag(session, module_info, strict=False):
"""
:param session : PDCClient instance
@@ -207,6 +207,7 @@ def get_module_tag(session, module_info, strict=False):
"""
return get_module(session, module_info, strict=strict)['koji_tag']
def get_module_modulemd(session, module_info, strict=False):
"""
:param session : PDCClient instance
@@ -223,17 +224,19 @@ def get_module_modulemd(session, module_info, strict=False):
if not yaml:
if strict:
raise ValueError("Failed to find modulemd entry in PDC for "
"%r" % module_info)
"%r" % module_info)
else:
return None
return _extract_modulemd(yaml, strict=strict)
def _extract_modulemd(yaml, strict=False):
mmd = modulemd.ModuleMetadata()
mmd.loads(yaml)
return mmd
def resolve_profiles(session, mmd, keys):
"""
:param session : PDCClient instance
@@ -266,6 +269,7 @@ def resolve_profiles(session, mmd, keys):
# Return the union of all rpms in all profiles of the given keys.
return results
def get_module_build_dependencies(session, module_info, strict=False):
"""
:param session : PDCClient instance
@@ -279,7 +283,6 @@ def get_module_build_dependencies(session, module_info, strict=False):
log.debug("get_module_build_dependencies(%r, strict=%r)" % (module_info, strict))
# XXX get definitive list of modules
deps = []
queried_module = get_module(session, module_info, strict=strict)
yaml = queried_module['modulemd']
queried_mmd = _extract_modulemd(yaml, strict=strict)
@@ -313,6 +316,7 @@ def get_module_build_dependencies(session, module_info, strict=False):
return module_tags
def get_module_commit_hash_and_version(session, module_info):
"""
Gets the commit hash and version of a module stored in PDC

1
module_build_service/scheduler/__init__.py
View File

@@ -9,6 +9,7 @@ import module_build_service.scheduler.consumer
import logging
log = logging.getLogger(__name__)
def main(initial_messages, stop_condition):
""" Run the consumer until some condition is met.

6
module_build_service/scheduler/consumer.py
View File

@@ -31,7 +31,6 @@ import itertools
import fedmsg.consumers
import moksha.hub
from module_build_service.errors import ValidationError
from module_build_service.utils import module_build_state_from_msg
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
@@ -47,9 +46,8 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer):
"""
topic = ['{}.{}.'.format(pref.rstrip('.'), cat)
for pref, cat
in itertools.product(
conf.messaging_topic_prefix,
module_build_service.messaging._messaging_backends[conf.messaging]['services'])]
in itertools.product(conf.messaging_topic_prefix,
module_build_service.messaging._messaging_backends[conf.messaging]['services'])]
if not topic:
topic = '*'
log.debug('Setting topics: {}'.format(', '.join(topic)))

12
module_build_service/scheduler/handlers/components.py
View File

@@ -68,10 +68,10 @@ def _finalize(config, session, msg, state):
parent = component_build.module_build
# If the macro build failed, then the module is doomed.
if (component_build.package == 'module-build-macros'
and state != koji.BUILD_STATES['COMPLETE']):
if (component_build.package == 'module-build-macros' and
state != koji.BUILD_STATES['COMPLETE']):
parent.transition(config, state=models.BUILD_STATES['failed'],
state_reason=state_reason)
state_reason=state_reason)
session.commit()
return
@@ -107,7 +107,7 @@ def _finalize(config, session, msg, state):
# change message here.
if failed_components_in_batch:
log.info("Batch done, but not tagging because of failed "
"component builds.")
"component builds.")
else:
log.info("Batch done. No component to tag")
further_work += [messaging.KojiRepoChange(
@@ -127,7 +127,7 @@ def _finalize(config, session, msg, state):
session.commit()
elif (any([c.state != koji.BUILD_STATES['BUILDING']
for c in unbuilt_components_in_batch])):
for c in unbuilt_components_in_batch])):
# We are not in the middle of the batch building and
# we have some unbuilt components in this batch. We might hit the
# concurrent builds threshold in previous call of continue_batch_build
@@ -144,8 +144,10 @@ def _finalize(config, session, msg, state):
def complete(config, session, msg):
return _finalize(config, session, msg, state=koji.BUILD_STATES['COMPLETE'])
def failed(config, session, msg):
return _finalize(config, session, msg, state=koji.BUILD_STATES['FAILED'])
def canceled(config, session, msg):
return _finalize(config, session, msg, state=koji.BUILD_STATES['CANCELED'])

19
module_build_service/scheduler/handlers/modules.py
View File

@@ -29,7 +29,7 @@ import module_build_service.pdc
import module_build_service.utils
import module_build_service.messaging
from module_build_service.utils import (
start_next_batch_build, attempt_to_reuse_all_components,
attempt_to_reuse_all_components,
get_rpm_release_from_mmd)
from module_build_service.builder.KojiContentGenerator import KojiContentGenerator
@@ -46,6 +46,7 @@ logging.basicConfig(level=logging.DEBUG)
def get_artifact_from_srpm(srpm_path):
return os.path.basename(srpm_path).replace(".src.rpm", "")
def failed(config, session, msg):
"""
Called whenever a module enters the 'failed' state.
@@ -66,8 +67,8 @@ def failed(config, session, msg):
unbuilt_components = [
c for c in build.component_builds
if (c.state != koji.BUILD_STATES['COMPLETE']
and c.state != koji.BUILD_STATES["FAILED"])
if (c.state != koji.BUILD_STATES['COMPLETE'] and
c.state != koji.BUILD_STATES["FAILED"])
]
if build.koji_tag:
@@ -120,6 +121,7 @@ def done(config, session, msg):
build_logs.stop(build.id)
def wait(config, session, msg):
""" Called whenever a module enters the 'wait' state.
@@ -161,9 +163,9 @@ def wait(config, session, msg):
if conf.system != "koji":
# In case of mock, we do not try to get anything from pdc,
# just generate our own koji_tag to identify the module in messages.
tag = '-'.join(['module', module_info['name'],
str(module_info['stream']), str(module_info['version'])])
tag = '-'.join(['module',
module_info['name'],
str(module_info['stream']), str(module_info['version'])])
for name, stream in build.mmd().buildrequires.items():
@@ -218,9 +220,6 @@ def wait(config, session, msg):
session.commit()
raise
groups = module_build_service.builder.GenericBuilder.default_buildroot_groups(
session, build)
log.debug("Found tag=%s for module %r" % (tag, build))
# Hang on to this information for later. We need to know which build is
# associated with which koji tag, so that when their repos are regenerated
@@ -279,7 +278,7 @@ def wait(config, session, msg):
if state == koji.BUILD_STATES['COMPLETE']:
if config.system == "koji":
log.info("module-build-macros is already built. "
"Regenerating the repo.")
"Regenerating the repo.")
task_id = builder.koji_session.newRepo(
builder.module_build_tag['name'])
build.new_repo_task_id = task_id

1
module_build_service/scheduler/handlers/tags.py
View File

@@ -28,7 +28,6 @@ import module_build_service.pdc
import logging
import koji
from module_build_service import models, log
from module_build_service.utils import start_next_batch_build
logging.basicConfig(level=logging.DEBUG)

14
module_build_service/scheduler/producer.py
View File

@@ -51,7 +51,7 @@ class MBSProducer(PollingProducer):
self.process_paused_module_builds(conf, session)
self.trigger_new_repo_when_stalled(conf, session)
self.delete_old_koji_targets(conf, session)
except Exception as e:
except Exception:
msg = 'Error in poller execution:'
log.exception(msg)
@@ -155,7 +155,7 @@ class MBSProducer(PollingProducer):
# First batch is number '1'.
for i in range(1, module_build.batch + 1):
n = len([c for c in module_build.component_builds
if c.batch == i ])
if c.batch == i])
log.info(' * {0} components in batch {1}'
.format(n, i))
@@ -189,14 +189,13 @@ class MBSProducer(PollingProducer):
# Check to see if module builds that are in build state but don't have
# any component builds being built can be worked on
for module_build in session.query(models.ModuleBuild).filter_by(
state=models.BUILD_STATES['build']).all():
for module_build in session.query(models.ModuleBuild).filter_by(state=models.BUILD_STATES['build']).all():
# If there are no components in the build state on the module build,
# then no possible event will start off new component builds.
# But do not try to start new builds when we are waiting for the
# repo-regen.
if (not module_build.current_batch(koji.BUILD_STATES['BUILDING'])
and not module_build.new_repo_task_id):
if (not module_build.current_batch(koji.BUILD_STATES['BUILDING']) and
not module_build.new_repo_task_id):
# Initialize the builder...
builder = GenericBuilder.create_from_module(
session, module_build, config)
@@ -224,8 +223,7 @@ class MBSProducer(PollingProducer):
koji_session = module_build_service.builder.KojiModuleBuilder\
.get_session(config, None)
for module_build in session.query(models.ModuleBuild).filter_by(
state=models.BUILD_STATES['build']).all():
for module_build in session.query(models.ModuleBuild).filter_by(state=models.BUILD_STATES['build']).all():
if not module_build.new_repo_task_id:
continue

4
module_build_service/scm.py
View File

@@ -27,8 +27,6 @@
"""SCM handler functions."""
from six.moves import http_client
import os
import subprocess as sp
import re
@@ -169,7 +167,7 @@ class SCM(object):
"The original message was: %s" % e.message)
raise
timestamp = SCM._run(["git", "show" , "-s", "--format=%ct"], chdir=sourcedir)[1]
timestamp = SCM._run(["git", "show", "-s", "--format=%ct"], chdir=sourcedir)[1]
dt = datetime.datetime.utcfromtimestamp(int(timestamp))
self.version = dt.strftime("%Y%m%d%H%M%S")
else:

63
module_build_service/utils.py
View File

@@ -29,11 +29,8 @@ import time
import shutil
import tempfile
import os
import logging
import copy
import kobo.rpmlib
import inspect
from six import iteritems
import hashlib
import modulemd
@@ -53,6 +50,7 @@ from module_build_service.pdc import get_module_commit_hash_and_version
import concurrent.futures
def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception):
""" A decorator that allows to retry a section of code...
...until success or timeout.
@@ -104,6 +102,7 @@ def at_concurrent_component_threshold(config, session):
return False
def start_build_component(builder, c):
"""
Submits single component build to builder. Called in thread
@@ -122,9 +121,10 @@ def start_build_component(builder, c):
if not c.task_id and c.state == koji.BUILD_STATES['BUILDING']:
c.state = koji.BUILD_STATES['FAILED']
c.state_reason = ("Failed to build artifact %s: "
"Builder did not return task ID" % (c.package))
"Builder did not return task ID" % (c.package))
return
def continue_batch_build(config, module, session, builder, components=None):
"""
Continues building current batch. Submits next components in the batch
@@ -140,10 +140,10 @@ def continue_batch_build(config, module, session, builder, components=None):
# successfully built yet or isn't currently being built.
unbuilt_components = components or [
c for c in module.component_builds
if (c.state != koji.BUILD_STATES['COMPLETE']
and c.state != koji.BUILD_STATES['BUILDING']
and c.state != koji.BUILD_STATES['FAILED']
and c.batch == module.batch)
if (c.state != koji.BUILD_STATES['COMPLETE'] and
c.state != koji.BUILD_STATES['BUILDING'] and
c.state != koji.BUILD_STATES['FAILED'] and
c.batch == module.batch)
]
if not unbuilt_components:
@@ -162,8 +162,8 @@ def continue_batch_build(config, module, session, builder, components=None):
session, module, c.package)
# If a component build can't be reused, we need to check
# the concurrent threshold.
if (not previous_component_build
and at_concurrent_component_threshold(config, session)):
if (not previous_component_build and
at_concurrent_component_threshold(config, session)):
log.info('Concurrent build threshold met')
break
@@ -195,8 +195,8 @@ def continue_batch_build(config, module, session, builder, components=None):
# We therefore have to generate fake KojiRepoChange message, because the
# repo has been also done in the past and build system will not send us
# any message now.
if (all(c.state in [koji.BUILD_STATES['COMPLETE'], koji.BUILD_STATES['FAILED']]
or c.reused_component_id
if (all(c.state in [koji.BUILD_STATES['COMPLETE'],
koji.BUILD_STATES['FAILED']] or c.reused_component_id
for c in unbuilt_components) and builder.module_build_tag):
further_work += [module_build_service.messaging.KojiRepoChange(
'start_build_batch: fake msg', builder.module_build_tag['name'])]
@@ -204,6 +204,7 @@ def continue_batch_build(config, module, session, builder, components=None):
session.commit()
return further_work
def start_next_batch_build(config, module, session, builder, components=None):
"""
Tries to start the build of next batch. In case there are still unbuilt
@@ -294,10 +295,10 @@ def start_next_batch_build(config, module, session, builder, components=None):
# successfully built yet or isn't currently being built.
unbuilt_components = components or [
c for c in module.component_builds
if (c.state != koji.BUILD_STATES['COMPLETE']
and c.state != koji.BUILD_STATES['BUILDING']
and c.state != koji.BUILD_STATES['FAILED']
and c.batch == module.batch)
if (c.state != koji.BUILD_STATES['COMPLETE'] and
c.state != koji.BUILD_STATES['BUILDING'] and
c.state != koji.BUILD_STATES['FAILED'] and
c.batch == module.batch)
]
# If there are no components to build, skip the batch and start building
@@ -308,11 +309,12 @@ def start_next_batch_build(config, module, session, builder, components=None):
return start_next_batch_build(config, module, session, builder)
log.info("Starting build of next batch %d, %s" % (module.batch,
unbuilt_components))
unbuilt_components))
return continue_batch_build(
config, module, session, builder, unbuilt_components)
def pagination_metadata(p_query, request_args):
"""
Returns a dictionary containing metadata about the paginated query.
@@ -358,6 +360,7 @@ def pagination_metadata(p_query, request_args):
return pagination_data
def filter_component_builds(flask_request):
"""
Returns a flask_sqlalchemy.Pagination object based on the request parameters
@@ -398,12 +401,13 @@ def filter_component_builds(flask_request):
query = query.order_by(column)
else:
raise ValidationError('An invalid order_by or order_desc_by key '
'was supplied')
'was supplied')
page = flask_request.args.get('page', 1, type=int)
per_page = flask_request.args.get('per_page', 10, type=int)
return query.paginate(page, per_page, False)
def filter_module_builds(flask_request):
"""
Returns a flask_sqlalchemy.Pagination object based on the request parameters
@@ -468,7 +472,7 @@ def filter_module_builds(flask_request):
query = query.order_by(column)
else:
raise ValidationError('An invalid order_by or order_desc_by key '
'was supplied')
'was supplied')
page = flask_request.args.get('page', 1, type=int)
per_page = flask_request.args.get('per_page', 10, type=int)
@@ -567,6 +571,7 @@ def _scm_get_latest(pkg):
'error': None
}
def format_mmd(mmd, scmurl):
"""
Prepares the modulemd for the MBS. This does things such as replacing the
@@ -667,6 +672,7 @@ def format_mmd(mmd, scmurl):
if err_msg:
raise UnprocessableEntity(err_msg)
def merge_included_mmd(mmd, included_mmd):
"""
Merges two modulemds. This merges only metadata which are needed in
@@ -678,6 +684,7 @@ def merge_included_mmd(mmd, included_mmd):
else:
mmd.xmd['mbs']['rpms'].update(included_mmd.xmd['mbs']['rpms'])
def record_component_builds(mmd, module, initial_batch=1,
previous_buildorder=None, main_mmd=None):
import koji # Placed here to avoid py2/py3 conflicts...
@@ -788,6 +795,7 @@ def submit_module_build_from_yaml(username, handle, optional_params=None):
_url_check_re = re.compile(r"^[^:/]+:.*$")
def submit_module_build_from_scm(username, url, branch, allow_local_url=False,
skiptests=False, optional_params=None):
# Translate local paths into file:// URL
@@ -805,7 +813,6 @@ def submit_module_build_from_scm(username, url, branch, allow_local_url=False,
def submit_module_build(username, url, mmd, scm, yaml, optional_params=None):
# Import it here, because SCM uses utils methods
# and fails to import them because of dep-chain.
import module_build_service.scm
module = models.ModuleBuild.query.filter_by(
name=mmd.name, stream=mmd.stream, version=str(mmd.version)).first()
@@ -816,8 +823,8 @@ def submit_module_build(username, url, mmd, scm, yaml, optional_params=None):
if module.state not in (models.BUILD_STATES['failed'],
models.BUILD_STATES['init']):
err_msg = ('Module (state=%s) already exists. '
'Only new build or resubmission of build in "init" or '
'"failed" state is allowed.' % module.state)
'Only new build or resubmission of build in "init" or '
'"failed" state is allowed.' % module.state)
log.error(err_msg)
raise Conflict(err_msg)
log.debug('Resuming existing module build %r' % module)
@@ -870,9 +877,9 @@ def scm_url_schemes(terse=False):
"""
scm_types = {
"git": ("git://", "git+http://", "git+https://",
"git+rsync://", "http://", "https://", "file://")
}
"git": ("git://", "git+http://", "git+https://",
"git+rsync://", "http://", "https://", "file://")
}
if not terse:
return scm_types
@@ -882,12 +889,14 @@ def scm_url_schemes(terse=False):
scheme_list.extend([scheme[:-3] for scheme in scm_schemes])
return list(set(scheme_list))
def get_scm_url_re():
schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True)))
return re.compile(
r"(?P<giturl>(?:(?P<scheme>(" + schemes_re + r"))://(?P<host>[^/]+))?"
r"(?P<repopath>/[^\?]+))\?(?P<modpath>[^#]*)#(?P<revision>.+)")
def module_build_state_from_msg(msg):
state = int(msg.module_build_state)
# TODO better handling
@@ -896,6 +905,7 @@ def module_build_state_from_msg(msg):
% (state, type(state), list(models.BUILD_STATES.values())))
return state
def reuse_component(component, previous_component_build,
change_state_now=False):
"""
@@ -943,6 +953,7 @@ def reuse_component(component, previous_component_build,
)
]
def attempt_to_reuse_all_components(builder, session, module):
"""
Tries to reuse all the components in a build. The components are also
@@ -986,6 +997,7 @@ def attempt_to_reuse_all_components(builder, session, module):
return True
def get_reusable_component(session, module, component_name):
"""
Returns the component (RPM) build of a module that can be reused
@@ -1221,6 +1233,7 @@ def validate_koji_tag(tag_arg_names, pre='', post='-', dict_key='name'):
return validation_decorator
def get_rpm_release_from_mmd(mmd):
"""
Returns the dist tag based on the modulemd metadata and MBS configuration.

11
module_build_service/views.py
View File

@@ -36,7 +36,7 @@ from module_build_service import models, db
from module_build_service.utils import (
pagination_metadata, filter_module_builds, filter_component_builds,
submit_module_build_from_scm, submit_module_build_from_yaml,
scm_url_schemes, get_scm_url_re)
get_scm_url_re)
from module_build_service.errors import (
ValidationError, Forbidden, NotFound)
@@ -69,6 +69,7 @@ api_v1 = {
},
}
class ComponentBuildAPI(MethodView):
def get(self, id):
@@ -206,8 +207,8 @@ class BaseHandler(object):
return {k: v for k, v in self.data.items() if k not in ["owner", "scmurl", "branch"]}
def validate_optional_params(self):
forbidden_params = [k for k in self.data if k not in models.ModuleBuild.__table__.columns
and k not in ["branch"]]
forbidden_params = [k for k in self.data if k not in models.ModuleBuild.__table__.columns and
k not in ["branch"]]
if forbidden_params:
raise ValidationError('The request contains unspecified parameters: {}'.format(", ".join(forbidden_params)))
@@ -289,11 +290,11 @@ def register_api_v1():
app.add_url_rule(val['url'],
endpoint=key,
view_func=module_view,
**val['options'])
**val['options'])
else:
app.add_url_rule(val['url'],
endpoint=key,
view_func=component_view,
**val['options'])
register_api_v1()

8
tests/__init__.py
View File

@@ -21,7 +21,6 @@
# Written by Matt Prahl <mprahl@redhat.com
import os
import copy
import module_build_service
from datetime import datetime, timedelta
@@ -29,13 +28,13 @@ from module_build_service import db
from module_build_service.config import init_config
from module_build_service.models import ModuleBuild, ComponentBuild, make_session
import modulemd
from module_build_service.utils import get_scm_url_re
import module_build_service.pdc
base_dir = os.path.dirname(__file__)
app = module_build_service.app
conf = init_config(app)
def init_data():
db.session.remove()
db.drop_all()
@@ -399,6 +398,7 @@ def test_reuse_component_init_data():
session.add(component_four_build_two)
session.commit()
def test_reuse_shared_userspace_init_data():
db.session.remove()
db.drop_all()
@@ -419,7 +419,7 @@ def test_reuse_shared_userspace_init_data():
build_one = module_build_service.models.ModuleBuild()
build_one.name = mmd.name
build_one.stream = mmd.stream
build_one.stream = mmd.stream
build_one.version = mmd.version
build_one.state = 5
build_one.modulemd = yaml
@@ -466,7 +466,7 @@ def test_reuse_shared_userspace_init_data():
build_one = module_build_service.models.ModuleBuild()
build_one.name = mmd.name
build_one.stream = mmd.stream
build_one.stream = mmd.stream
build_one.version = mmd.version
build_one.state = 3
build_one.modulemd = yaml

45
tests/test_auth.py
View File

@@ -20,8 +20,7 @@
#
# Written by Ralph Bean <rbean@redhat.com>
from nose.tools import raises, eq_
from nose.tools import eq_
import unittest
import mock
@@ -42,10 +41,10 @@ class TestAuthModule(unittest.TestCase):
request.cookies.return_value = {}
with self.assertRaises(module_build_service.errors.Unauthorized) as cm:
result = module_build_service.auth.get_user(request)
module_build_service.auth.get_user(request)
self.assertEquals(str(cm.exception),
"No 'authorization' header found.")
"No 'authorization' header found.")
@patch('module_build_service.auth._get_token_info')
@patch('module_build_service.auth._get_user_info')
@@ -57,10 +56,10 @@ class TestAuthModule(unittest.TestCase):
# https://www.youtube.com/watch?v=G-LtddOgUCE
name = "Joey Jo Jo Junior Shabadoo"
mocked_get_token_info = {"active": False, "username": name,
"scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"}
"scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"}
get_token_info.return_value = mocked_get_token_info
get_user_info.return_value = {"groups":["group"]}
get_user_info.return_value = {"groups": ["group"]}
headers = {"authorization": "Bearer foobar"}
request = mock.MagicMock()
@@ -70,10 +69,10 @@ class TestAuthModule(unittest.TestCase):
request.headers.__contains__.side_effect = headers.__contains__
with self.assertRaises(module_build_service.errors.Unauthorized) as cm:
result = module_build_service.auth.get_user(request)
module_build_service.auth.get_user(request)
self.assertEquals(str(cm.exception),
"OIDC token invalid or expired.")
"OIDC token invalid or expired.")
@patch('module_build_service.auth._get_token_info')
@patch('module_build_service.auth._get_user_info')
@@ -85,10 +84,10 @@ class TestAuthModule(unittest.TestCase):
# https://www.youtube.com/watch?v=G-LtddOgUCE
name = "Joey Jo Jo Junior Shabadoo"
mocked_get_token_info = {"active": True, "username": name,
"scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"}
"scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"}
get_token_info.return_value = mocked_get_token_info
get_user_info.return_value = {"groups":["group"]}
get_user_info.return_value = {"groups": ["group"]}
headers = {"authorization": "Bearer foobar"}
request = mock.MagicMock()
@@ -115,7 +114,7 @@ class TestAuthModule(unittest.TestCase):
module_build_service.auth.get_user(request)
self.assertEquals(str(cm.exception),
"OIDC_CLIENT_SECRETS must be set in server config.")
"OIDC_CLIENT_SECRETS must be set in server config.")
@patch('module_build_service.auth._get_token_info')
@patch('module_build_service.auth._get_user_info')
@@ -126,11 +125,12 @@ class TestAuthModule(unittest.TestCase):
'OIDC_REQUIRED_SCOPE': 'mbs-scope'}):
# https://www.youtube.com/watch?v=G-LtddOgUCE
name = "Joey Jo Jo Junior Shabadoo"
mocked_get_token_info = {"active": True, "username": name,
"scope": "openid https://id.fedoraproject.org/scope/groups"}
mocked_get_token_info = {"active": True,
"username": name,
"scope": "openid https://id.fedoraproject.org/scope/groups"}
get_token_info.return_value = mocked_get_token_info
get_user_info.return_value = {"groups":["group"]}
get_user_info.return_value = {"groups": ["group"]}
headers = {"authorization": "Bearer foobar"}
request = mock.MagicMock()
@@ -140,11 +140,11 @@ class TestAuthModule(unittest.TestCase):
request.headers.__contains__.side_effect = headers.__contains__
with self.assertRaises(module_build_service.errors.Unauthorized) as cm:
result = module_build_service.auth.get_user(request)
module_build_service.auth.get_user(request)
self.assertEquals(str(cm.exception),
"Required OIDC scope 'mbs-scope' not present: "
"['openid', 'https://id.fedoraproject.org/scope/groups']")
"Required OIDC scope 'mbs-scope' not present: "
"['openid', 'https://id.fedoraproject.org/scope/groups']")
@patch('module_build_service.auth._get_token_info')
@patch('module_build_service.auth._get_user_info')
@@ -154,11 +154,12 @@ class TestAuthModule(unittest.TestCase):
with patch.dict('module_build_service.app.config', {'OIDC_CLIENT_SECRETS': client_secrets}):
# https://www.youtube.com/watch?v=G-LtddOgUCE
name = "Joey Jo Jo Junior Shabadoo"
mocked_get_token_info = {"active": True, "username": name,
"scope": "openid https://id.fedoraproject.org/scope/groups"}
mocked_get_token_info = {"active": True,
"username": name,
"scope": "openid https://id.fedoraproject.org/scope/groups"}
get_token_info.return_value = mocked_get_token_info
get_user_info.return_value = {"groups":["group"]}
get_user_info.return_value = {"groups": ["group"]}
headers = {"authorization": "Bearer foobar"}
request = mock.MagicMock()
@@ -168,7 +169,7 @@ class TestAuthModule(unittest.TestCase):
request.headers.__contains__.side_effect = headers.__contains__
with self.assertRaises(module_build_service.errors.Forbidden) as cm:
result = module_build_service.auth.get_user(request)
module_build_service.auth.get_user(request)
self.assertEquals(str(cm.exception),
"OIDC_REQUIRED_SCOPE must be set in server config.")
"OIDC_REQUIRED_SCOPE must be set in server config.")

75
tests/test_build/test_build.py
View File

@@ -24,7 +24,7 @@ import unittest
import koji
import vcr
import os
from os import path, mkdir, remove
from os import path, mkdir
from os.path import dirname
from shutil import copyfile
@@ -38,7 +38,6 @@ from module_build_service import db, models, conf, build_logs
from mock import patch, PropertyMock
from tests import app, init_data, test_reuse_component_init_data
import os
import json
import itertools
@@ -51,6 +50,7 @@ cassette_dir = base_dir + '/vcr-request-data/'
user = ('Homer J. Simpson', set(['packager']))
class MockedSCM(object):
def __init__(self, mocked_scm, name, mmd_filename, commit=None):
self.mocked_scm = mocked_scm
@@ -70,13 +70,14 @@ class MockedSCM(object):
mkdir(scm_dir)
base_dir = path.abspath(path.dirname(__file__))
copyfile(path.join(base_dir, '..', 'staged_data', self.mmd_filename),
path.join(scm_dir, self.mmd_filename))
path.join(scm_dir, self.mmd_filename))
return scm_dir
def get_latest(self, branch='master'):
return branch
class TestModuleBuilder(GenericBuilder):
"""
Test module builder which succeeds for every build.
@@ -173,7 +174,7 @@ class TestModuleBuilder(GenericBuilder):
module_build_service.scheduler.consumer.work_queue_put(msg)
def build(self, artifact_name, source):
print "Starting building artifact %s: %s" % (artifact_name, source)
print("Starting building artifact %s: %s" % (artifact_name, source))
TestModuleBuilder._build_id += 1
@@ -207,18 +208,18 @@ class TestModuleBuilder(GenericBuilder):
@patch("module_build_service.config.Config.system",
new_callable=PropertyMock, return_value="test")
new_callable=PropertyMock, return_value="test")
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={
'srpm-build':
set(['shadow-utils', 'fedora-release', 'redhat-rpm-config',
'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']),
'build':
set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk',
'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash',
'info', 'bzip2', 'grep', 'redhat-rpm-config',
'diffutils', 'make', 'patch', 'shadow-utils',
'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])})
'srpm-build':
set(['shadow-utils', 'fedora-release', 'redhat-rpm-config',
'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']),
'build':
set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk',
'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash',
'info', 'bzip2', 'grep', 'redhat-rpm-config',
'diffutils', 'make', 'patch', 'shadow-utils',
'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])})
class TestBuild(unittest.TestCase):
# Global variable used for tests if needed
@@ -300,7 +301,7 @@ class TestBuild(unittest.TestCase):
# or "ready" state.
for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]])
# All components has to be tagged, so tag_groups and buildroot_groups are empty...
self.assertEqual(tag_groups, [])
@@ -323,13 +324,13 @@ class TestBuild(unittest.TestCase):
return json.loads(rv.data)
with patch("module_build_service.config.Config.yaml_submit_allowed",
new_callable=PropertyMock, return_value=True):
new_callable=PropertyMock, return_value=True):
conf.set_item("yaml_submit_allowed", True)
data = submit()
self.assertEqual(data['id'], 1)
with patch("module_build_service.config.Config.yaml_submit_allowed",
new_callable=PropertyMock, return_value=False):
new_callable=PropertyMock, return_value=False):
data = submit()
self.assertEqual(data['status'], 403)
self.assertEqual(data['message'], 'YAML submission is not enabled')
@@ -373,11 +374,11 @@ class TestBuild(unittest.TestCase):
# method. We just cancel the build here using the web API to simulate
# user cancelling the build in the middle of building.
def on_build_cb(cls, artifact_name, source):
self.client.patch(
'/module-build-service/1/module-builds/' + str(module_build_id),
data=json.dumps({'state': 'failed'}))
self.client.patch('/module-build-service/1/module-builds/' + str(module_build_id),
data=json.dumps({'state': 'failed'}))
cancelled_tasks = []
def on_cancel_cb(cls, task_id):
cancelled_tasks.append(task_id)
@@ -433,7 +434,7 @@ class TestBuild(unittest.TestCase):
# or "ready" state.
for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]])
@timed(30)
@patch('module_build_service.auth.get_user', return_value=user)
@@ -465,7 +466,7 @@ class TestBuild(unittest.TestCase):
main_stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
over_threshold = conf.num_concurrent_builds < \
db.session.query(models.ComponentBuild).filter_by(
state=koji.BUILD_STATES['BUILDING']).count()
state=koji.BUILD_STATES['BUILDING']).count()
return main_stop(message) or over_threshold
msgs = []
@@ -477,7 +478,7 @@ class TestBuild(unittest.TestCase):
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
# When this fails, it can mean that num_concurrent_builds
# threshold has been met.
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]])
@timed(30)
@patch('module_build_service.auth.get_user', return_value=user)
@@ -600,7 +601,7 @@ class TestBuild(unittest.TestCase):
@patch("module_build_service.config.Config.num_concurrent_builds",
new_callable=PropertyMock, return_value=1)
def test_all_builds_in_batch_fail(self, conf_num_concurrent_builds, mocked_scm,
mocked_get_user, conf_system, dbg):
mocked_get_user, conf_system, dbg):
"""
Tests that if the build in batch fails, other components in a batch
are still build, but next batch is not started.
@@ -661,8 +662,8 @@ class TestBuild(unittest.TestCase):
tag_groups = []
tag_groups.append(set(
['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357',
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
def on_tag_artifacts_cb(cls, artifacts):
self.assertEqual(tag_groups.pop(0), set(artifacts))
@@ -671,8 +672,9 @@ class TestBuild(unittest.TestCase):
buildtag_groups = []
buildtag_groups.append(set(
['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357',
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
self.assertEqual(buildtag_groups.pop(0), set(artifacts))
TestModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb
@@ -697,7 +699,7 @@ class TestBuild(unittest.TestCase):
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_build_reuse_all_without_build_macros(self, mocked_scm, mocked_get_user,
conf_system, dbg):
conf_system, dbg):
"""
Tests that we can reuse components even when the reused module does
not have module-build-macros component.
@@ -718,8 +720,8 @@ class TestBuild(unittest.TestCase):
tag_groups = []
tag_groups.append(set(
['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357',
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
def on_tag_artifacts_cb(cls, artifacts):
self.assertEqual(tag_groups.pop(0), set(artifacts))
@@ -728,8 +730,9 @@ class TestBuild(unittest.TestCase):
buildtag_groups = []
buildtag_groups.append(set(
['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357',
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357',
'tangerine-0.22-3.module_testmodule_master_20170109091357']))
def on_buildroot_add_artifacts_cb(cls, artifacts, install):
self.assertEqual(buildtag_groups.pop(0), set(artifacts))
TestModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb
@@ -742,7 +745,7 @@ class TestBuild(unittest.TestCase):
# or "ready" state.
for build in models.ComponentBuild.query.filter_by(module_id=2).all():
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]])
self.assertNotEqual(build.package, "module-build-macros")
@timed(60)
@@ -769,7 +772,7 @@ class TestBuild(unittest.TestCase):
# Set the components from batch 2 to COMPLETE
components = models.ComponentBuild.query.filter_by(module_id=module_build_id)
for c in components:
print c
print(c)
if c.batch == 2:
c.state = koji.BUILD_STATES["COMPLETE"]
db.session.commit()
@@ -783,4 +786,4 @@ class TestBuild(unittest.TestCase):
# or "ready" state.
for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]])

3
tests/test_builder/test_koji.py
View File

@@ -36,6 +36,7 @@ from tests import conf, init_data
from module_build_service.builder import KojiModuleBuilder
class FakeKojiModuleBuilder(KojiModuleBuilder):
@module_build_service.utils.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError))
@@ -50,6 +51,7 @@ class FakeKojiModuleBuilder(KojiModuleBuilder):
return koji_session
class TestKojiBuilder(unittest.TestCase):
def setUp(self):
@@ -234,4 +236,3 @@ class TestGetKojiClientSession(unittest.TestCase):
components=[])
args, kwargs = mocked_krb_login.call_args
self.assertTrue(set([('proxyuser', self.module.owner)]).issubset(set(kwargs.items())))

6
tests/test_config.py
View File

@@ -20,16 +20,12 @@
#
# Written by Jan Kaluza <jkaluza@redhat.com>
from nose.tools import raises, eq_
import unittest
import mock
import os.path
from mock import patch
from module_build_service import conf
class TestConfig(unittest.TestCase):
def test_path_expanduser(self):
test_dir = "~/modulebuild/builds"

12
tests/test_content_generator.py
View File

@@ -28,10 +28,8 @@ import os
from os import path
from os.path import dirname
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
import module_build_service.utils
from module_build_service import models, conf, build_logs
from mock import patch, Mock
@@ -60,7 +58,6 @@ class TestBuild(unittest.TestCase):
self.vcr = vcr.use_cassette(filename)
self.vcr.__enter__()
def tearDown(self):
# Necessary to restart the twisted reactor for the next test.
import sys
@@ -88,8 +85,8 @@ class TestBuild(unittest.TestCase):
pkg_res.return_value = Mock()
pkg_res.return_value.version = "current-tested-version"
rpm_mock = Mock()
rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \
"rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"
rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \
"rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"
attrs = {'communicate.return_value': (rpm_out, 'error'),
'wait.return_value': 0}
rpm_mock.configure_mock(**attrs)
@@ -127,8 +124,8 @@ class TestBuild(unittest.TestCase):
pkg_res.return_value = Mock()
pkg_res.return_value.version = "current-tested-version"
rpm_mock = Mock()
rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \
"rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"
rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \
"rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"
attrs = {'communicate.return_value': (rpm_out, 'error'),
'wait.return_value': 0}
rpm_mock.configure_mock(**attrs)
@@ -149,7 +146,6 @@ class TestBuild(unittest.TestCase):
rpms_in_tag.assert_called_once()
self.assertEqual(expected_output, ret)
def test_prepare_file_directory(self):
""" Test preparation of directory with output files """
dir_path = self.cg._prepare_file_directory()

1
tests/test_logger.py
View File

@@ -29,7 +29,6 @@ import unittest
from module_build_service import log
from module_build_service.logger import ModuleBuildLogs
from module_build_service.scheduler.consumer import MBSConsumer
from mock import patch, PropertyMock
class TestLogger(unittest.TestCase):

5
tests/test_messaging.py
View File

@@ -22,7 +22,7 @@
import unittest
from module_build_service import messaging, conf
from module_build_service import messaging
from mock import patch, PropertyMock
@@ -55,7 +55,7 @@ class TestFedmsgMessaging(unittest.TestCase):
self.assertEqual(msg.build_new_state, 1)
@patch("module_build_service.config.Config.system",
new_callable=PropertyMock, return_value="copr")
new_callable=PropertyMock, return_value="copr")
def test_copr_build_end(self, conf_system):
# http://fedora-fedmsg.readthedocs.io/en/latest/topics.html#copr-build-end
copr_build_end_msg = {
@@ -116,7 +116,6 @@ class TestFedmsgMessaging(unittest.TestCase):
self.assertEqual(msg.tag, "module-debugging-tools-master-20170405115403-build")
self.assertEqual(msg.artifact, "module-build-macros")
def test_buildsys_repo_done(self):
# https://fedora-fedmsg.readthedocs.io/en/latest/topics.html#id134
buildsys_tag_msg = {

3
tests/test_pdc.py
View File

@@ -34,6 +34,7 @@ import modulemd
base_dir = os.path.dirname(__file__)
cassette_dir = base_dir + '/vcr-request-data/'
class TestPDCModule(unittest.TestCase):
def setUp(self):
@@ -111,5 +112,5 @@ class TestPDCModule(unittest.TestCase):
set(['shadow-utils', 'redhat-rpm-config', 'rpm-build',
'fedora-modular-release', 'fedpkg-minimal', 'gnupg2',
'bash'])
}
}
self.assertEqual(result, expected)

12
tests/test_scheduler/test_module_wait.py
View File

@@ -30,7 +30,7 @@ import os
import vcr
import koji
from tests import conf, db, app, scheduler_init_data
from module_build_service import conf, build_logs
from module_build_service import build_logs
base_dir = os.path.dirname(os.path.dirname(__file__))
cassette_dir = base_dir + '/vcr-request-data/'
@@ -85,11 +85,11 @@ class TestModuleWait(unittest.TestCase):
from_module_event.return_value = mocked_module_build
msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1,
module_build_state='some state')
module_build_state='some state')
self.fn(config=self.config, session=self.session, msg=msg)
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={'build': [], 'srpm-build': []})
return_value={'build': [], 'srpm-build': []})
@patch("module_build_service.builder.KojiModuleBuilder.get_session")
@patch("module_build_service.builder.GenericBuilder.create_from_module")
@patch('module_build_service.pdc')
@@ -113,13 +113,13 @@ class TestModuleWait(unittest.TestCase):
create_builder.return_value = builder
msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1,
module_build_state='some state')
module_build_state='some state')
module_build_service.scheduler.handlers.modules.wait(
config=conf, session=db.session, msg=msg)
koji_session.newRepo.assert_called_once_with("module-123-build")
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={'build': [], 'srpm-build': []})
return_value={'build': [], 'srpm-build': []})
@patch("module_build_service.builder.KojiModuleBuilder.get_session")
@patch("module_build_service.builder.GenericBuilder.create_from_module")
@patch('module_build_service.pdc')
@@ -143,7 +143,7 @@ class TestModuleWait(unittest.TestCase):
create_builder.return_value = builder
msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1,
module_build_state='some state')
module_build_state='some state')
module_build_service.scheduler.handlers.modules.wait(
config=conf, session=db.session, msg=msg)
self.assertTrue(not koji_session.newRepo.called)

14
tests/test_scheduler/test_poller.py
View File

@@ -20,19 +20,11 @@
import unittest
from os import path
import vcr
import modulemd
from mock import patch
import module_build_service.utils
import module_build_service.scm
from module_build_service import models, conf
from module_build_service.errors import ProgrammingError, ValidationError
from tests import test_reuse_component_init_data, init_data, db
import mock
from mock import PropertyMock
import koji
import module_build_service.scheduler.handlers.components
from module_build_service.builder import GenericBuilder, KojiModuleBuilder
from module_build_service.scheduler.producer import MBSProducer
import six.moves.queue as queue
from datetime import datetime, timedelta
@@ -41,6 +33,7 @@ BASE_DIR = path.abspath(path.dirname(__file__))
CASSETTES_DIR = path.join(
path.abspath(path.dirname(__file__)), '..', 'vcr-request-data')
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={'build': [], 'srpm-build': []})
@patch("module_build_service.scheduler.consumer.get_global_consumer")
@@ -123,10 +116,9 @@ class TestPoller(unittest.TestCase):
koji_session.newRepo.assert_called_once_with("module-testmodule-build")
def test_trigger_new_repo_when_succeded(self, create_builder,
koji_get_session, global_consumer,
dbg):
koji_get_session, global_consumer,
dbg):
"""
Tests that we do not call koji_sesion.newRepo when newRepo task
succeeded.

1
tests/test_scheduler/test_repo_done.py
View File

@@ -34,6 +34,7 @@ from tests import conf, db, app
base_dir = dirname(dirname(__file__))
cassette_dir = base_dir + '/vcr-request-data/'
class TestRepoDone(unittest.TestCase):
def setUp(self):

15
tests/test_scheduler/test_tag_tagged.py
View File

@@ -20,10 +20,8 @@
#
# Written by Jan Kaluza <jkaluza@redhat.com>
from os.path import dirname
import unittest
import mock
import vcr
from mock import patch
@@ -31,10 +29,11 @@ import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
import module_build_service.models
from tests import test_reuse_component_init_data
from tests import conf, db, app
from tests import conf, db
import koji
class TestTagTagged(unittest.TestCase):
def setUp(self):
@@ -63,9 +62,8 @@ class TestTagTagged(unittest.TestCase):
module_build_service.scheduler.handlers.tags.tagged(
config=conf, session=db.session, msg=msg)
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={'build': [], 'srpm-build': []})
return_value={'build': [], 'srpm-build': []})
@patch("module_build_service.builder.KojiModuleBuilder.get_session")
@patch("module_build_service.builder.GenericBuilder.create_from_module")
def test_newrepo(self, create_builder, koji_get_session, dbg):
@@ -123,9 +121,8 @@ class TestTagTagged(unittest.TestCase):
# status later in poller.
self.assertEqual(module_build.new_repo_task_id, 123456)
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={'build': [], 'srpm-build': []})
return_value={'build': [], 'srpm-build': []})
@patch("module_build_service.builder.KojiModuleBuilder.get_session")
@patch("module_build_service.builder.GenericBuilder.create_from_module")
def test_newrepo_still_building_components(self, create_builder, koji_get_session, dbg):
@@ -161,7 +158,7 @@ class TestTagTagged(unittest.TestCase):
self.assertTrue(not koji_session.newRepo.called)
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={'build': [], 'srpm-build': []})
return_value={'build': [], 'srpm-build': []})
@patch("module_build_service.builder.KojiModuleBuilder.get_session")
@patch("module_build_service.builder.GenericBuilder.create_from_module")
def test_newrepo_failed_components(self, create_builder, koji_get_session, dbg):
@@ -215,7 +212,7 @@ class TestTagTagged(unittest.TestCase):
self.assertEqual(module_build.new_repo_task_id, 123456)
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value = {'build': [], 'srpm-build': []})
return_value={'build': [], 'srpm-build': []})
@patch("module_build_service.builder.KojiModuleBuilder.get_session")
@patch("module_build_service.builder.GenericBuilder.create_from_module")
def test_newrepo_multiple_batches_tagged(

23
tests/test_utils/test_utils.py
View File

@@ -31,7 +31,6 @@ from module_build_service.errors import ProgrammingError, ValidationError
from tests import (test_reuse_component_init_data, init_data, db,
test_reuse_shared_userspace_init_data)
import mock
from mock import PropertyMock
import koji
import module_build_service.scheduler.handlers.components
from module_build_service.builder import GenericBuilder, KojiModuleBuilder
@@ -41,6 +40,7 @@ BASE_DIR = path.abspath(path.dirname(__file__))
CASSETTES_DIR = path.join(
path.abspath(path.dirname(__file__)), '..', 'vcr-request-data')
class MockedSCM(object):
def __init__(self, mocked_scm, name, mmd_filename, commit=None):
self.mocked_scm = mocked_scm
@@ -60,13 +60,14 @@ class MockedSCM(object):
mkdir(scm_dir)
base_dir = path.abspath(path.dirname(__file__))
copyfile(path.join(base_dir, '..', 'staged_data', self.mmd_filename),
path.join(scm_dir, self.name + ".yaml"))
path.join(scm_dir, self.name + ".yaml"))
return scm_dir
def get_latest(self, branch='master'):
return self.commit if self.commit else branch
class TestUtils(unittest.TestCase):
def setUp(self):
@@ -115,8 +116,8 @@ class TestUtils(unittest.TestCase):
'stream': 'master',
'version': '20170404161234'}},
'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'},
'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'},
'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}},
'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'},
'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}},
'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/testmodule'
'.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4',
}
@@ -133,6 +134,7 @@ class TestUtils(unittest.TestCase):
'f24': '4ceea43add2366d8b8c5a622a2fb563b625b9abf',
'f23': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c',
'f25': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'}
def mocked_get_latest(branch="master"):
return hashes_returned[branch]
mocked_scm.return_value.get_latest = mocked_get_latest
@@ -152,8 +154,8 @@ class TestUtils(unittest.TestCase):
'stream': 'master',
'version': '20170404161234'}},
'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'},
'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'},
'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}},
'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'},
'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}},
'scmurl': None,
}
}
@@ -368,7 +370,7 @@ class TestUtils(unittest.TestCase):
self.assertEquals(
validate_koji_tag_good_tag_value_in_dict_nondefault_key(
{'nondefault': 'module-foo'}), True)
{'nondefault': 'module-foo'}), True)
def test_validate_koji_tag_double_trouble_good(self):
""" Test that we pass on a list of tags that are good. """
@@ -413,7 +415,7 @@ class TestUtils(unittest.TestCase):
component states properly.
"""
MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
with app.app_context():
test_reuse_component_init_data()
# Mark the module build as failed, so we can resubmit it.
@@ -502,7 +504,6 @@ class DummyModuleBuilder(GenericBuilder):
self.tag_name = tag_name
self.config = config
def buildroot_connect(self, groups):
pass
@@ -548,6 +549,7 @@ class DummyModuleBuilder(GenericBuilder):
def list_tasks_for_components(self, component_builds=None, state='active'):
pass
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []})
class TestBatches(unittest.TestCase):
@@ -639,9 +641,6 @@ class TestBatches(unittest.TestCase):
builder = mock.MagicMock()
builder.buildroot_ready.return_value = False
further_work = module_build_service.utils.start_next_batch_build(
conf, module_build, db.session, builder)
# Batch number should not increase.
self.assertEqual(module_build.batch, 1)

43
tests/test_views/test_views.py
View File

@@ -28,7 +28,7 @@ import vcr
import modulemd as _modulemd
import module_build_service.scm
from mock import patch, Mock, PropertyMock, MagicMock
from mock import patch, PropertyMock, MagicMock
from shutil import copyfile
from os import path, mkdir
from os.path import dirname
@@ -96,7 +96,7 @@ class MockedSCM(object):
mkdir(scm_dir)
base_dir = path.abspath(path.dirname(__file__))
copyfile(path.join(base_dir, '..', 'staged_data', mmd_filename),
path.join(scm_dir, self.name + ".yaml"))
path.join(scm_dir, self.name + ".yaml"))
self.checkout_id += 1
@@ -258,7 +258,7 @@ class TestViews(unittest.TestCase):
def test_query_builds_filter_task_id(self):
rv = self.client.get('/module-build-service/1/component-builds/?task_id=12312346')
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], 1)
self.assertEquals(data['meta']['total'], 1)
def test_query_builds_filter_name(self):
rv = self.client.get('/module-build-service/1/module-builds/?name=nginx')
@@ -335,7 +335,7 @@ class TestViews(unittest.TestCase):
def test_query_builds_order_by(self):
rv = self.client.get('/module-build-service/1/module-builds/?'
'per_page=10&order_by=id')
'per_page=10&order_by=id')
items = json.loads(rv.data)['items']
# Check that the id is 1, 2, 3, ..., 10
for idx, item in enumerate(items):
@@ -343,7 +343,7 @@ class TestViews(unittest.TestCase):
def test_query_builds_order_desc_by(self):
rv = self.client.get('/module-build-service/1/module-builds/?'
'per_page=10&order_desc_by=id')
'per_page=10&order_desc_by=id')
items = json.loads(rv.data)['items']
# Check that the id is items[0]["id"], items[0]["id"] - 1, ...
for idx, item in enumerate(items):
@@ -355,7 +355,7 @@ class TestViews(unittest.TestCase):
we prefer order_desc_by.
"""
rv = self.client.get('/module-build-service/1/module-builds/?'
'per_page=10&order_desc_by=id&order_by=name')
'per_page=10&order_desc_by=id&order_by=name')
items = json.loads(rv.data)['items']
# Check that the id is items[0]["id"], items[0]["id"] - 1, ...
for idx, item in enumerate(items):
@@ -363,7 +363,7 @@ class TestViews(unittest.TestCase):
def test_query_builds_order_by_wrong_key(self):
rv = self.client.get('/module-build-service/1/module-builds/?'
'per_page=10&order_by=unknown')
'per_page=10&order_by=unknown')
data = json.loads(rv.data)
self.assertEquals(data['status'], 400)
self.assertEquals(data['error'], 'Bad Request')
@@ -472,7 +472,7 @@ class TestViews(unittest.TestCase):
{'branch': 'master', 'scmurl': 'git://badurl.com'}))
data = json.loads(rv.data)
self.assertEquals(data['message'], 'The submitted scmurl '
'git://badurl.com is not allowed')
'git://badurl.com is not allowed')
self.assertEquals(data['status'], 403)
self.assertEquals(data['error'], 'Forbidden')
@@ -483,15 +483,15 @@ class TestViews(unittest.TestCase):
'testmodule.git'}))
data = json.loads(rv.data)
self.assertEquals(data['message'], 'The submitted scmurl '
'git://pkgs.stg.fedoraproject.org/modules/testmodule.git '
'is not valid')
'git://pkgs.stg.fedoraproject.org/modules/testmodule.git '
'is not valid')
self.assertEquals(data['status'], 403)
self.assertEquals(data['error'], 'Forbidden')
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_build_bad_modulemd(self, mocked_scm, mocked_get_user):
mocked_scm_obj = MockedSCM(mocked_scm, "bad", "bad.yaml")
MockedSCM(mocked_scm, "bad", "bad.yaml")
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
@@ -562,8 +562,9 @@ class TestViews(unittest.TestCase):
@patch("module_build_service.config.Config.modules_allow_repository",
new_callable=PropertyMock, return_value=True)
def test_submit_build_includedmodule(self, conf, mocked_scm, mocked_get_user):
mocked_scm_obj = MockedSCM(mocked_scm, "includedmodules",
["includedmodules.yaml", "testmodule.yaml"])
MockedSCM(mocked_scm, "includedmodules", ["includedmodules.yaml",
"testmodule.yaml"])
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
@@ -609,9 +610,9 @@ class TestViews(unittest.TestCase):
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_build_includedmodule_custom_repo_not_allowed(self,
mocked_scm, mocked_get_user):
mocked_scm_obj = MockedSCM(mocked_scm, "includedmodules",
["includedmodules.yaml", "testmodule.yaml"])
mocked_scm, mocked_get_user):
MockedSCM(mocked_scm, "includedmodules", ["includedmodules.yaml",
"testmodule.yaml"])
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
@@ -651,9 +652,9 @@ class TestViews(unittest.TestCase):
return_value=('sammy', set(["packager", "mbs-admin"])))
def test_cancel_build_admin(self, mocked_get_user):
with patch("module_build_service.config.Config.admin_groups",
new_callable=PropertyMock, return_value=set(["mbs-admin"])):
new_callable=PropertyMock, return_value=set(["mbs-admin"])):
rv = self.client.patch('/module-build-service/1/module-builds/30',
data=json.dumps({'state': 'failed'}))
data=json.dumps({'state': 'failed'}))
data = json.loads(rv.data)
self.assertEquals(data['state'], 4)
@@ -663,9 +664,9 @@ class TestViews(unittest.TestCase):
return_value=('sammy', set(["packager"])))
def test_cancel_build_no_admin(self, mocked_get_user):
with patch("module_build_service.config.Config.admin_groups",
new_callable=PropertyMock, return_value=set(["mbs-admin"])):
new_callable=PropertyMock, return_value=set(["mbs-admin"])):
rv = self.client.patch('/module-build-service/1/module-builds/30',
data=json.dumps({'state': 'failed'}))
data=json.dumps({'state': 'failed'}))
data = json.loads(rv.data)
self.assertEquals(data['status'], 403)
@@ -861,7 +862,7 @@ class TestViews(unittest.TestCase):
def submit(scmurl):
return self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'scmurl': scmurl}))
{'branch': 'master', 'scmurl': scmurl}))
allow_custom_scmurls.return_value = False
res1 = submit('git://some.custom.url.org/modules/testmodule.git?#68931c9')