From 7ef297598475a035aa55d438e074a1a1f05bc4b0 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 21 Jun 2017 15:24:50 +0200 Subject: [PATCH 1/2] PEP8 fixes --- bootstrap/fix-json.py | 7 +- bootstrap/import-to-pdc.py | 1 - conf/config.py | 5 +- fedmsg.d/module_build_service.py | 2 +- module_build_service/__init__.py | 7 +- module_build_service/auth.py | 18 +++-- .../builder/CoprModuleBuilder.py | 2 - .../builder/KojiContentGenerator.py | 10 +-- .../builder/KojiModuleBuilder.py | 48 ++++++------ .../builder/MockModuleBuilder.py | 11 ++- module_build_service/builder/__init__.py | 2 +- module_build_service/builder/base.py | 2 + module_build_service/builder/utils.py | 4 +- module_build_service/config.py | 7 +- module_build_service/logger.py | 5 +- module_build_service/manage.py | 11 +-- module_build_service/messaging.py | 2 + module_build_service/models.py | 8 +- module_build_service/pdc.py | 22 +++--- module_build_service/scheduler/__init__.py | 1 + module_build_service/scheduler/consumer.py | 6 +- .../scheduler/handlers/components.py | 12 +-- .../scheduler/handlers/modules.py | 19 +++-- .../scheduler/handlers/tags.py | 1 - module_build_service/scheduler/producer.py | 14 ++-- module_build_service/scm.py | 4 +- module_build_service/utils.py | 63 +++++++++------- module_build_service/views.py | 11 +-- tests/__init__.py | 8 +- tests/test_auth.py | 45 +++++------ tests/test_build/test_build.py | 75 ++++++++++--------- tests/test_builder/test_koji.py | 3 +- tests/test_config.py | 6 +- tests/test_content_generator.py | 12 +-- tests/test_logger.py | 1 - tests/test_messaging.py | 5 +- tests/test_pdc.py | 3 +- tests/test_scheduler/test_module_wait.py | 12 +-- tests/test_scheduler/test_poller.py | 14 +--- tests/test_scheduler/test_repo_done.py | 1 + tests/test_scheduler/test_tag_tagged.py | 15 ++-- tests/test_utils/test_utils.py | 23 +++--- tests/test_views/test_views.py | 43 +++++------ 43 files changed, 282 insertions(+), 289 deletions(-) diff --git a/bootstrap/fix-json.py b/bootstrap/fix-json.py index ed071b0c..635b351a 100644 --- a/bootstrap/fix-json.py +++ b/bootstrap/fix-json.py @@ -1,17 +1,12 @@ import json -import requests -import os -import sys import yaml -import pdc_client - filename = 'bootstrap-master-1.json' print("Reading %s" % filename) with open(filename, 'r') as f: entry = json.loads(f.read()) -print entry['modulemd'] +print(entry['modulemd']) modulemd = yaml.load(entry['modulemd']) mbs = {} mbs['commit'] = modulemd['data']['xmd']['mbs_commit'] diff --git a/bootstrap/import-to-pdc.py b/bootstrap/import-to-pdc.py index 4a3b53ba..63d8fb2f 100644 --- a/bootstrap/import-to-pdc.py +++ b/bootstrap/import-to-pdc.py @@ -1,5 +1,4 @@ import json -import requests import os import sys diff --git a/conf/config.py b/conf/config.py index cfc165d4..ed54c60b 100644 --- a/conf/config.py +++ b/conf/config.py @@ -5,7 +5,7 @@ from os import path confdir = path.abspath(path.dirname(__file__)) # use parent dir as dbdir else fallback to current dir dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \ - else confdir + else confdir class BaseConfiguration(object): @@ -67,7 +67,7 @@ class BaseConfiguration(object): ALLOWED_GROUPS = set([ 'packager', - #'modularity-wg', + # 'modularity-wg', ]) # Available backends are: console, file, journal. @@ -172,6 +172,7 @@ class TestConfiguration(BaseConfiguration): class ProdConfiguration(BaseConfiguration): pass + class LocalBuildConfiguration(BaseConfiguration): LOG_LEVEL = 'debug' MESSAGING = 'in_memory' diff --git a/fedmsg.d/module_build_service.py b/fedmsg.d/module_build_service.py index 1e115543..3dbe3820 100644 --- a/fedmsg.d/module_build_service.py +++ b/fedmsg.d/module_build_service.py @@ -13,7 +13,7 @@ config = { "endpoints": { "fedora-infrastructure": [ # Just listen to staging for now, not to production (spam!) - #"tcp://hub.fedoraproject.org:9940", + # "tcp://hub.fedoraproject.org:9940", "tcp://stg.fedoraproject.org:9940", ], }, diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index 856d3451..a0a5d059 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -59,6 +59,7 @@ conf = init_config(app) db = SQLAlchemy(app) + @app.errorhandler(ValidationError) def validationerror_error(e): """Flask error handler for ValidationError exceptions""" @@ -104,6 +105,7 @@ init_logging(conf) log = getLogger(__name__) build_logs = ModuleBuildLogs(conf.build_logs_dir) + def get_url_for(*args, **kwargs): """ flask.url_for wrapper which creates the app_context on-the-fly. @@ -116,8 +118,9 @@ def get_url_for(*args, **kwargs): app.config['SERVER_NAME'] = 'localhost' with app.app_context(): log.debug("WARNING: get_url_for() has been called without the Flask " - "app_context. That can lead to SQLAlchemy errors caused by " - "multiple session being used in the same time.") + "app_context. That can lead to SQLAlchemy errors caused by " + "multiple session being used in the same time.") return url_for(*args, **kwargs) + from module_build_service import views diff --git a/module_build_service/auth.py b/module_build_service/auth.py index b4169342..58b87f20 100644 --- a/module_build_service/auth.py +++ b/module_build_service/auth.py @@ -37,18 +37,20 @@ def _json_loads(content): client_secrets = None + def _load_secrets(): global client_secrets if client_secrets: return - if not "OIDC_CLIENT_SECRETS" in app.config: + if "OIDC_CLIENT_SECRETS" not in app.config: raise Forbidden("OIDC_CLIENT_SECRETS must be set in server config.") secrets = _json_loads(open(app.config['OIDC_CLIENT_SECRETS'], - 'r').read()) + 'r').read()) client_secrets = list(secrets.values())[0] + def _get_token_info(token): """ Asks the token_introspection_uri for the validity of a token. @@ -57,9 +59,9 @@ def _get_token_info(token): return None request = {'token': token, - 'token_type_hint': 'Bearer', - 'client_id': client_secrets['client_id'], - 'client_secret': client_secrets['client_secret']} + 'token_type_hint': 'Bearer', + 'client_id': client_secrets['client_id'], + 'client_secret': client_secrets['client_secret']} headers = {'Content-type': 'application/x-www-form-urlencoded'} resp = requests.post(client_secrets['token_introspection_uri'], data=request, headers=headers) @@ -89,7 +91,7 @@ def get_user(request): _load_secrets() - if not "authorization" in request.headers: + if "authorization" not in request.headers: raise Unauthorized("No 'authorization' header found.") header = request.headers['authorization'].strip() @@ -105,10 +107,10 @@ def get_user(request): log.exception(error) raise Exception(error) - if not data or not "active" in data or not data["active"]: + if not data or "active" not in data or not data["active"]: raise Unauthorized("OIDC token invalid or expired.") - if not "OIDC_REQUIRED_SCOPE" in app.config: + if "OIDC_REQUIRED_SCOPE" not in app.config: raise Forbidden("OIDC_REQUIRED_SCOPE must be set in server config.") presented_scopes = data['scope'].split(' ') diff --git a/module_build_service/builder/CoprModuleBuilder.py b/module_build_service/builder/CoprModuleBuilder.py index ba085adf..ccbdce7b 100644 --- a/module_build_service/builder/CoprModuleBuilder.py +++ b/module_build_service/builder/CoprModuleBuilder.py @@ -35,7 +35,6 @@ from copr.client import CoprClient from copr.exceptions import CoprRequestException from module_build_service import log -from module_build_service.models import ModuleBuild import module_build_service.scm import module_build_service.utils @@ -70,7 +69,6 @@ class CoprModuleBuilder(GenericBuilder): self.chroot = "custom-1-x86_64" self.__prep = False - @classmethod def _get_client(cls, config): return CoprClient.create_from_file_config(config.copr_config) diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 83d8682f..078cc1f8 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -57,7 +57,6 @@ class KojiContentGenerator(object): self.mmd = module.modulemd self.config = config - def __repr__(self): return "" % (self.module_name) @@ -185,7 +184,7 @@ class KojiContentGenerator(object): try: rpms, builds = session.listTaggedRPMS(tag, latest=True) - except koji.GenericError as e: + except koji.GenericError: log.exception("Failed to list rpms in tag %r", tag) # If the tag doesn't exist.. then there are no rpms in that tag. return [] @@ -244,7 +243,6 @@ class KojiContentGenerator(object): } return ret - def _get_output(self, output_path): ret = [] rpms = self._koji_rpms_in_tag(self.module.koji_tag) @@ -302,7 +300,6 @@ class KojiContentGenerator(object): return ret - def _get_content_generator_metadata(self, output_path): ret = { u"metadata_version": 0, @@ -327,11 +324,10 @@ class KojiContentGenerator(object): log_path = os.path.join(prepdir, "build.log") try: shutil.copy(build_logs.path(self.module.id), log_path) - except IOError, e: + except IOError as e: log.exception(e) return prepdir - def koji_import(self): """This method imports given module into the configured koji instance as a content generator based build @@ -345,7 +341,7 @@ class KojiContentGenerator(object): build_info = session.CGImport(metadata, file_dir) log.debug("Content generator import done: %s", json.dumps(build_info, sort_keys=True, indent=4)) - except Exception, e: + except Exception as e: log.exception("Content generator import failed: %s", e) raise e finally: diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index bc24dfa3..285516c6 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -40,7 +40,7 @@ import threading import munch from OpenSSL.SSL import SysCallError -from module_build_service import conf, log, db +from module_build_service import log import module_build_service.scm import module_build_service.utils from module_build_service.builder.utils import execute_cmd @@ -80,9 +80,9 @@ class KojiModuleBuilder(GenericBuilder): raise ValueError("No koji_arches specified in the config.") # These eventually get populated by calling _connect and __prep is set to True - self.module_tag = None # string - self.module_build_tag = None # string - self.module_target = None # A koji target dict + self.module_tag = None # string + self.module_build_tag = None # string + self.module_target = None # A koji target dict self.build_priority = config.koji_build_priority self.components = components @@ -127,11 +127,10 @@ class KojiModuleBuilder(GenericBuilder): log.info("%r buildroot is not yet ready.. wait." % self) return ready - @staticmethod def get_disttag_srpm(disttag, module_build): - #Taken from Karsten's create-distmacro-pkg.sh + # Taken from Karsten's create-distmacro-pkg.sh # - however removed any provides to system-release/redhat-release name = 'module-build-macros' @@ -284,7 +283,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules # Create or update individual tags self.module_tag = self._koji_create_tag( - self.tag_name, self.arches, perm="admin") # the main tag needs arches so pungi can dump it + self.tag_name, self.arches, perm="admin") # the main tag needs arches so pungi can dump it self.module_build_tag = self._koji_create_tag( self.tag_name + "-build", self.arches, perm="admin") @@ -370,7 +369,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules log.info("Waiting for task_id=%s to finish" % task_id) - timeout = 60 * 60 # 60 minutes + timeout = 60 * 60 # 60 minutes + @module_build_service.utils.retry(timeout=timeout, wait_on=koji.GenericError) def get_result(): log.debug("Waiting for task_id=%s to finish" % task_id) @@ -467,9 +467,9 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules source, task_info['task_id'], self)) return task_info['task_id'], koji.BUILD_STATES['COMPLETE'], 'Build already exists.', task_info['nvr'] - self._koji_whitelist_packages([artifact_name,]) + self._koji_whitelist_packages([artifact_name]) if '://' not in source: - #treat source as an srpm and upload it + # treat source as an srpm and upload it serverdir = _unique_path('cli-build') callback = None self.koji_session.uploadWrapper(source, serverdir, callback=callback) @@ -480,8 +480,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules # The reason is that it is faster to build this RPM in # already existing shared target, because Koji does not need to do # repo-regen. - if (artifact_name == "module-build-macros" - and self.config.koji_build_macros_target): + if (artifact_name == "module-build-macros" and + self.config.koji_build_macros_target): module_target = self.config.koji_build_macros_target else: module_target = self.module_target['name'] @@ -536,13 +536,14 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules priority = 0 if inheritance_data: priority = inheritance_data[-1]['priority'] + 10 + def record_exists(parent_id, data): for item in data: if parent_id == item['parent_id']: return True return False - for parent in parent_tags: # We expect that they're sorted + for parent in parent_tags: # We expect that they're sorted parent = self._get_tag(parent) if record_exists(parent['id'], inheritance_data): continue @@ -571,10 +572,10 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules raise ValueError("Expected dict {'group' : [str(package1), ...]") dest_tag = self._get_tag(dest_tag)['name'] - existing_groups = dict([ - (p['name'], p['group_id']) - for p in self.koji_session.getTagGroups(dest_tag, inherit=False) - ]) + existing_groups = dict([(p['name'], p['group_id']) + for p + in self.koji_session.getTagGroups(dest_tag, inherit=False) + ]) for group, packages in groups.items(): group_id = existing_groups.get(group, None) @@ -589,7 +590,6 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules for pkg in packages: self.koji_session.groupPackageListAdd(dest_tag, group, pkg) - @module_build_service.utils.validate_koji_tag('tag_name') def _koji_create_tag(self, tag_name, arches=None, perm=None): """ @@ -613,8 +613,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules raise ValueError("Expected list or None on input got %s" % type(arches)) current_arches = [] - if taginfo['arches']: # None if none - current_arches = taginfo['arches'].split() # string separated by empty spaces + if taginfo['arches']: # None if none + current_arches = taginfo['arches'].split() # string separated by empty spaces if set(arches) != set(current_arches): opts['arches'] = " ".join(arches) @@ -629,7 +629,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules raise ValueError("Unknown permissions %s" % perm) perm_id = perm_ids[perm] - if taginfo['perm'] not in (perm_id, perm): # check either id or the string + if taginfo['perm'] not in (perm_id, perm): # check either id or the string opts['perm'] = perm_id opts['extra'] = { @@ -638,7 +638,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules # edit tag with opts self.koji_session.editTag2(tag_name, **opts) - return self._get_tag(tag_name) # Return up2date taginfo + return self._get_tag(tag_name) # Return up2date taginfo def _koji_whitelist_packages(self, packages, tags=None): if not tags: @@ -681,7 +681,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules if not target_info: target_info = self.koji_session.createBuildTarget(name, build_tag['name'], dest_tag['name']) - else: # verify whether build and destination tag matches + else: # verify whether build and destination tag matches if build_tag['name'] != target_info['build_tag_name']: raise SystemError("Target references unexpected build_tag_name. Got '%s', expected '%s'. Please contact administrator." % (target_info['build_tag_name'], build_tag['name'])) if dest_tag['name'] != target_info['dest_tag_name']: @@ -725,7 +725,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules # TODO: https://pagure.io/fm-orchestrator/issue/397 # Subj: Do not mix target/tag when looking for component builds if (c.package == task_opts['mbs_artifact_name'] and - c.module_build.koji_tag == task_opts['mbs_module_target']): + c.module_build.koji_tag == task_opts['mbs_module_target']): tasks.append(task) return tasks diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 1eb062a3..29ad5efc 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -28,10 +28,9 @@ import koji import kobo.rpmlib import modulemd import pipes -import yaml import threading -from module_build_service import conf, log, db +from module_build_service import conf, log import module_build_service.scm import module_build_service.utils import module_build_service.scheduler @@ -104,7 +103,7 @@ mdpolicy=group:primary self.tag_name = tag_name self.config = config self.groups = [] - self.arch = "x86_64" # TODO: We may need to change that in the future + self.arch = "x86_64" # TODO: We may need to change that in the future self.yum_conf = MockModuleBuilder.MOCK_YUM_CONF_TEMPLATE # Create main directory for this tag @@ -183,7 +182,7 @@ mdpolicy=group:primary pkglist_f.close() mmd_path = os.path.join(path, "modules.yaml") - modulemd.dump_all(mmd_path, [ m1_mmd ]) + modulemd.dump_all(mmd_path, [m1_mmd]) # Generate repo and inject modules.yaml there. execute_cmd(['/usr/bin/createrepo_c', '--pkglist', pkglist, path]) @@ -477,8 +476,8 @@ class SCMBuilder(BaseBuilder): # this code. distgit_get_branch = \ "sh -c {}'; git -C {} checkout {}'".format(pipes.quote(distgit_get), - artifact_name, - branch) + artifact_name, + branch) f.writelines([ "config_opts['scm'] = True\n", diff --git a/module_build_service/builder/__init__.py b/module_build_service/builder/__init__.py index be80f471..ae135a43 100644 --- a/module_build_service/builder/__init__.py +++ b/module_build_service/builder/__init__.py @@ -1,6 +1,7 @@ from module_build_service import conf from base import GenericBuilder from KojiModuleBuilder import KojiModuleBuilder +from MockModuleBuilder import MockModuleBuilder __all__ = [ GenericBuilder @@ -9,7 +10,6 @@ __all__ = [ GenericBuilder.register_backend_class(KojiModuleBuilder) -from MockModuleBuilder import MockModuleBuilder GenericBuilder.register_backend_class(MockModuleBuilder) if conf.system == "copr": diff --git a/module_build_service/builder/base.py b/module_build_service/builder/base.py index 499cb918..6882d368 100644 --- a/module_build_service/builder/base.py +++ b/module_build_service/builder/base.py @@ -60,6 +60,8 @@ Koji workflow 8) (optional) wait for selected builds to be available in buildroot """ + + class GenericBuilder(six.with_metaclass(ABCMeta)): """ External Api for builders diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index fdbbd07f..0f8d301b 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -8,7 +8,7 @@ import logging import urlgrabber.grabber as grabber import urlgrabber.progress as progress import module_build_service -from module_build_service import log, scm, messaging +from module_build_service import log logging.basicConfig(level=logging.DEBUG) @@ -75,7 +75,7 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None): # Get the list of all RPMs and builds in a tag. try: rpms, builds = session.listTaggedRPMS(tag, latest=True) - except koji.GenericError as e: + except koji.GenericError: log.exception("Failed to list rpms in tag %r" % tag) # Reformat builds so they are dict with build_id as a key. diff --git a/module_build_service/config.py b/module_build_service/config.py index ff8768f8..f0ef9765 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -109,6 +109,7 @@ def init_config(app): app.config.from_object(config_section_obj) return conf + class Path: """ Config type for paths. Expands the users home directory. @@ -189,14 +190,14 @@ class Config(object): 'desc': 'List of allowed koji tag prefixes.'}, 'koji_target_delete_time': { 'type': int, - 'default': 24*3600, + 'default': 24 * 3600, 'desc': 'Time in seconds after which the Koji target of ' - 'built module is deleted'}, + 'built module is deleted'}, 'koji_enable_content_generator': { 'type': bool, 'default': True, 'desc': 'Enable or disable imports to koji using content ' - 'generator api'}, + 'generator api'}, 'allow_custom_scmurls': { 'type': bool, 'default': False, diff --git a/module_build_service/logger.py b/module_build_service/logger.py index c48b8ee4..f5407d05 100644 --- a/module_build_service/logger.py +++ b/module_build_service/logger.py @@ -53,6 +53,7 @@ levels["info"] = logging.INFO log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' + class ModuleBuildFileHandler(logging.FileHandler): """ FileHandler subclass which handles only messages generated during @@ -138,14 +139,16 @@ def str_to_log_level(level): Available levels are: debug, info, warning, error """ - if not level in levels: + if level not in levels: return logging.NOTSET return levels[level] + def supported_log_backends(): return ("console", "journal", "file") + def init_logging(conf): """ Initializes logging according to configuration file. diff --git a/module_build_service/manage.py b/module_build_service/manage.py index 2c94f873..6357a088 100644 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -30,20 +30,11 @@ import ssl from shutil import rmtree import getpass -import fedmsg.config -import moksha.hub -import moksha.hub.hub -import moksha.hub.reactor - from module_build_service import app, conf, db from module_build_service import models -from module_build_service.pdc import ( - get_pdc_client_session, get_module, - get_module_tag, get_module_build_dependencies) from module_build_service.utils import ( submit_module_build_from_scm, ) -from module_build_service.messaging import MBSModule import module_build_service.messaging import module_build_service.scheduler.consumer @@ -154,7 +145,7 @@ def build_module_locally(url, branch, skiptests=False): username = getpass.getuser() submit_module_build_from_scm(username, url, branch, allow_local_url=True, - skiptests=skiptests) + skiptests=skiptests) stop = module_build_service.scheduler.make_simple_stop_condition(db.session) diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index 2d9c5e05..1ef10d79 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -265,6 +265,7 @@ class KojiBuildChange(BaseMessage): self.module_build_id = module_build_id self.state_reason = state_reason + class KojiTagChange(BaseMessage): """ A class that inherits from BaseMessage to provide a message @@ -277,6 +278,7 @@ class KojiTagChange(BaseMessage): self.tag = tag self.artifact = artifact + class KojiRepoChange(BaseMessage): """ A class that inherits from BaseMessage to provide a message object for a repo's info (in fedmsg this replaces the msg dictionary) diff --git a/module_build_service/models.py b/module_build_service/models.py index 6786968f..ed7712a4 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -67,6 +67,7 @@ BUILD_STATES = { INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()} + @contextlib.contextmanager def _dummy_context_mgr(): """ @@ -75,6 +76,7 @@ def _dummy_context_mgr(): """ yield None + @contextlib.contextmanager def make_session(conf): """ @@ -142,7 +144,7 @@ class ModuleBuild(MBSBase): if not self.batch: raise ValueError("No batch is in progress: %r" % self.batch) - if state != None: + if state is not None: return [ component for component in self.component_builds if component.batch == self.batch and component.state == state @@ -162,7 +164,7 @@ class ModuleBuild(MBSBase): if not self.batch: raise ValueError("No batch is in progress: %r" % self.batch) - if state != None: + if state is not None: return [ component for component in self.component_builds if component.batch <= self.batch and component.state == state @@ -339,7 +341,7 @@ class ModuleBuild(MBSBase): 'state_reason': self.state_reason, "owner": self.owner, "name": self.name, - "scmurl": self.scmurl, + "scmurl": self.scmurl, "time_submitted": self._utc_datetime_to_iso(self.time_submitted), "time_modified": self._utc_datetime_to_iso(self.time_modified), "time_completed": self._utc_datetime_to_iso(self.time_completed), diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index b39edbd0..8c7b0ea8 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -28,13 +28,11 @@ import modulemd from pdc_client import PDCClient -import copy import inspect import pprint import logging -log = logging.getLogger() - import six +log = logging.getLogger() def get_pdc_client_session(config): @@ -57,6 +55,7 @@ def get_pdc_client_session(config): insecure=config.pdc_insecure, ) + def get_variant_dict(data): """ :param data: one of following @@ -117,7 +116,6 @@ def get_variant_dict(data): if 'variant_type' not in result.keys(): result['variant_type'] = 'module' - elif is_module_dict(data): result = {'variant_id': data['name'], 'variant_version': data['version']} @@ -144,13 +142,14 @@ def variant_dict_from_str(module_str): release_start = module_str.rfind('-') version_start = module_str.rfind('-', 0, release_start) - module_info['variant_release'] = module_str[release_start+1:] - module_info['variant_version'] = module_str[version_start+1:release_start] + module_info['variant_release'] = module_str[release_start + 1:] + module_info['variant_version'] = module_str[version_start + 1:release_start] module_info['variant_id'] = module_str[:version_start] module_info['variant_type'] = 'module' return module_info + def get_module(session, module_info, strict=False): """ :param session : PDCClient instance @@ -173,7 +172,7 @@ def get_module(session, module_info, strict=False): if module_info.get('active'): query['active'] = module_info['active'] - retval = session['unreleasedvariants/'](page_size=-1, **query) # ordering=variant_release... + retval = session['unreleasedvariants/'](page_size=-1, **query) # ordering=variant_release... # Error handling if not retval: @@ -197,6 +196,7 @@ def get_module(session, module_info, strict=False): return module + def get_module_tag(session, module_info, strict=False): """ :param session : PDCClient instance @@ -207,6 +207,7 @@ def get_module_tag(session, module_info, strict=False): """ return get_module(session, module_info, strict=strict)['koji_tag'] + def get_module_modulemd(session, module_info, strict=False): """ :param session : PDCClient instance @@ -223,17 +224,19 @@ def get_module_modulemd(session, module_info, strict=False): if not yaml: if strict: raise ValueError("Failed to find modulemd entry in PDC for " - "%r" % module_info) + "%r" % module_info) else: return None return _extract_modulemd(yaml, strict=strict) + def _extract_modulemd(yaml, strict=False): mmd = modulemd.ModuleMetadata() mmd.loads(yaml) return mmd + def resolve_profiles(session, mmd, keys): """ :param session : PDCClient instance @@ -266,6 +269,7 @@ def resolve_profiles(session, mmd, keys): # Return the union of all rpms in all profiles of the given keys. return results + def get_module_build_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance @@ -279,7 +283,6 @@ def get_module_build_dependencies(session, module_info, strict=False): log.debug("get_module_build_dependencies(%r, strict=%r)" % (module_info, strict)) # XXX get definitive list of modules - deps = [] queried_module = get_module(session, module_info, strict=strict) yaml = queried_module['modulemd'] queried_mmd = _extract_modulemd(yaml, strict=strict) @@ -313,6 +316,7 @@ def get_module_build_dependencies(session, module_info, strict=False): return module_tags + def get_module_commit_hash_and_version(session, module_info): """ Gets the commit hash and version of a module stored in PDC diff --git a/module_build_service/scheduler/__init__.py b/module_build_service/scheduler/__init__.py index 3c81d6bf..bfbf7592 100644 --- a/module_build_service/scheduler/__init__.py +++ b/module_build_service/scheduler/__init__.py @@ -9,6 +9,7 @@ import module_build_service.scheduler.consumer import logging log = logging.getLogger(__name__) + def main(initial_messages, stop_condition): """ Run the consumer until some condition is met. diff --git a/module_build_service/scheduler/consumer.py b/module_build_service/scheduler/consumer.py index 41667cc8..67b3300a 100644 --- a/module_build_service/scheduler/consumer.py +++ b/module_build_service/scheduler/consumer.py @@ -31,7 +31,6 @@ import itertools import fedmsg.consumers import moksha.hub -from module_build_service.errors import ValidationError from module_build_service.utils import module_build_state_from_msg import module_build_service.messaging import module_build_service.scheduler.handlers.repos @@ -47,9 +46,8 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer): """ topic = ['{}.{}.'.format(pref.rstrip('.'), cat) for pref, cat - in itertools.product( - conf.messaging_topic_prefix, - module_build_service.messaging._messaging_backends[conf.messaging]['services'])] + in itertools.product(conf.messaging_topic_prefix, + module_build_service.messaging._messaging_backends[conf.messaging]['services'])] if not topic: topic = '*' log.debug('Setting topics: {}'.format(', '.join(topic))) diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index da3d42ac..3a6788a7 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -68,10 +68,10 @@ def _finalize(config, session, msg, state): parent = component_build.module_build # If the macro build failed, then the module is doomed. - if (component_build.package == 'module-build-macros' - and state != koji.BUILD_STATES['COMPLETE']): + if (component_build.package == 'module-build-macros' and + state != koji.BUILD_STATES['COMPLETE']): parent.transition(config, state=models.BUILD_STATES['failed'], - state_reason=state_reason) + state_reason=state_reason) session.commit() return @@ -107,7 +107,7 @@ def _finalize(config, session, msg, state): # change message here. if failed_components_in_batch: log.info("Batch done, but not tagging because of failed " - "component builds.") + "component builds.") else: log.info("Batch done. No component to tag") further_work += [messaging.KojiRepoChange( @@ -127,7 +127,7 @@ def _finalize(config, session, msg, state): session.commit() elif (any([c.state != koji.BUILD_STATES['BUILDING'] - for c in unbuilt_components_in_batch])): + for c in unbuilt_components_in_batch])): # We are not in the middle of the batch building and # we have some unbuilt components in this batch. We might hit the # concurrent builds threshold in previous call of continue_batch_build @@ -144,8 +144,10 @@ def _finalize(config, session, msg, state): def complete(config, session, msg): return _finalize(config, session, msg, state=koji.BUILD_STATES['COMPLETE']) + def failed(config, session, msg): return _finalize(config, session, msg, state=koji.BUILD_STATES['FAILED']) + def canceled(config, session, msg): return _finalize(config, session, msg, state=koji.BUILD_STATES['CANCELED']) diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 421d7998..ffcf3527 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -29,7 +29,7 @@ import module_build_service.pdc import module_build_service.utils import module_build_service.messaging from module_build_service.utils import ( - start_next_batch_build, attempt_to_reuse_all_components, + attempt_to_reuse_all_components, get_rpm_release_from_mmd) from module_build_service.builder.KojiContentGenerator import KojiContentGenerator @@ -46,6 +46,7 @@ logging.basicConfig(level=logging.DEBUG) def get_artifact_from_srpm(srpm_path): return os.path.basename(srpm_path).replace(".src.rpm", "") + def failed(config, session, msg): """ Called whenever a module enters the 'failed' state. @@ -66,8 +67,8 @@ def failed(config, session, msg): unbuilt_components = [ c for c in build.component_builds - if (c.state != koji.BUILD_STATES['COMPLETE'] - and c.state != koji.BUILD_STATES["FAILED"]) + if (c.state != koji.BUILD_STATES['COMPLETE'] and + c.state != koji.BUILD_STATES["FAILED"]) ] if build.koji_tag: @@ -120,6 +121,7 @@ def done(config, session, msg): build_logs.stop(build.id) + def wait(config, session, msg): """ Called whenever a module enters the 'wait' state. @@ -161,9 +163,9 @@ def wait(config, session, msg): if conf.system != "koji": # In case of mock, we do not try to get anything from pdc, # just generate our own koji_tag to identify the module in messages. - tag = '-'.join(['module', module_info['name'], - str(module_info['stream']), str(module_info['version'])]) - + tag = '-'.join(['module', + module_info['name'], + str(module_info['stream']), str(module_info['version'])]) for name, stream in build.mmd().buildrequires.items(): @@ -218,9 +220,6 @@ def wait(config, session, msg): session.commit() raise - groups = module_build_service.builder.GenericBuilder.default_buildroot_groups( - session, build) - log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is # associated with which koji tag, so that when their repos are regenerated @@ -279,7 +278,7 @@ def wait(config, session, msg): if state == koji.BUILD_STATES['COMPLETE']: if config.system == "koji": log.info("module-build-macros is already built. " - "Regenerating the repo.") + "Regenerating the repo.") task_id = builder.koji_session.newRepo( builder.module_build_tag['name']) build.new_repo_task_id = task_id diff --git a/module_build_service/scheduler/handlers/tags.py b/module_build_service/scheduler/handlers/tags.py index 3bd208f0..493a5da9 100644 --- a/module_build_service/scheduler/handlers/tags.py +++ b/module_build_service/scheduler/handlers/tags.py @@ -28,7 +28,6 @@ import module_build_service.pdc import logging import koji from module_build_service import models, log -from module_build_service.utils import start_next_batch_build logging.basicConfig(level=logging.DEBUG) diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index e78881bf..dec72b5f 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -51,7 +51,7 @@ class MBSProducer(PollingProducer): self.process_paused_module_builds(conf, session) self.trigger_new_repo_when_stalled(conf, session) self.delete_old_koji_targets(conf, session) - except Exception as e: + except Exception: msg = 'Error in poller execution:' log.exception(msg) @@ -155,7 +155,7 @@ class MBSProducer(PollingProducer): # First batch is number '1'. for i in range(1, module_build.batch + 1): n = len([c for c in module_build.component_builds - if c.batch == i ]) + if c.batch == i]) log.info(' * {0} components in batch {1}' .format(n, i)) @@ -189,14 +189,13 @@ class MBSProducer(PollingProducer): # Check to see if module builds that are in build state but don't have # any component builds being built can be worked on - for module_build in session.query(models.ModuleBuild).filter_by( - state=models.BUILD_STATES['build']).all(): + for module_build in session.query(models.ModuleBuild).filter_by(state=models.BUILD_STATES['build']).all(): # If there are no components in the build state on the module build, # then no possible event will start off new component builds. # But do not try to start new builds when we are waiting for the # repo-regen. - if (not module_build.current_batch(koji.BUILD_STATES['BUILDING']) - and not module_build.new_repo_task_id): + if (not module_build.current_batch(koji.BUILD_STATES['BUILDING']) and + not module_build.new_repo_task_id): # Initialize the builder... builder = GenericBuilder.create_from_module( session, module_build, config) @@ -224,8 +223,7 @@ class MBSProducer(PollingProducer): koji_session = module_build_service.builder.KojiModuleBuilder\ .get_session(config, None) - for module_build in session.query(models.ModuleBuild).filter_by( - state=models.BUILD_STATES['build']).all(): + for module_build in session.query(models.ModuleBuild).filter_by(state=models.BUILD_STATES['build']).all(): if not module_build.new_repo_task_id: continue diff --git a/module_build_service/scm.py b/module_build_service/scm.py index 266cdbd5..e4244477 100644 --- a/module_build_service/scm.py +++ b/module_build_service/scm.py @@ -27,8 +27,6 @@ """SCM handler functions.""" -from six.moves import http_client - import os import subprocess as sp import re @@ -169,7 +167,7 @@ class SCM(object): "The original message was: %s" % e.message) raise - timestamp = SCM._run(["git", "show" , "-s", "--format=%ct"], chdir=sourcedir)[1] + timestamp = SCM._run(["git", "show", "-s", "--format=%ct"], chdir=sourcedir)[1] dt = datetime.datetime.utcfromtimestamp(int(timestamp)) self.version = dt.strftime("%Y%m%d%H%M%S") else: diff --git a/module_build_service/utils.py b/module_build_service/utils.py index a9788559..d6bbba0b 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -29,11 +29,8 @@ import time import shutil import tempfile import os -import logging -import copy import kobo.rpmlib import inspect -from six import iteritems import hashlib import modulemd @@ -53,6 +50,7 @@ from module_build_service.pdc import get_module_commit_hash_and_version import concurrent.futures + def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception): """ A decorator that allows to retry a section of code... ...until success or timeout. @@ -104,6 +102,7 @@ def at_concurrent_component_threshold(config, session): return False + def start_build_component(builder, c): """ Submits single component build to builder. Called in thread @@ -122,9 +121,10 @@ def start_build_component(builder, c): if not c.task_id and c.state == koji.BUILD_STATES['BUILDING']: c.state = koji.BUILD_STATES['FAILED'] c.state_reason = ("Failed to build artifact %s: " - "Builder did not return task ID" % (c.package)) + "Builder did not return task ID" % (c.package)) return + def continue_batch_build(config, module, session, builder, components=None): """ Continues building current batch. Submits next components in the batch @@ -140,10 +140,10 @@ def continue_batch_build(config, module, session, builder, components=None): # successfully built yet or isn't currently being built. unbuilt_components = components or [ c for c in module.component_builds - if (c.state != koji.BUILD_STATES['COMPLETE'] - and c.state != koji.BUILD_STATES['BUILDING'] - and c.state != koji.BUILD_STATES['FAILED'] - and c.batch == module.batch) + if (c.state != koji.BUILD_STATES['COMPLETE'] and + c.state != koji.BUILD_STATES['BUILDING'] and + c.state != koji.BUILD_STATES['FAILED'] and + c.batch == module.batch) ] if not unbuilt_components: @@ -162,8 +162,8 @@ def continue_batch_build(config, module, session, builder, components=None): session, module, c.package) # If a component build can't be reused, we need to check # the concurrent threshold. - if (not previous_component_build - and at_concurrent_component_threshold(config, session)): + if (not previous_component_build and + at_concurrent_component_threshold(config, session)): log.info('Concurrent build threshold met') break @@ -195,8 +195,8 @@ def continue_batch_build(config, module, session, builder, components=None): # We therefore have to generate fake KojiRepoChange message, because the # repo has been also done in the past and build system will not send us # any message now. - if (all(c.state in [koji.BUILD_STATES['COMPLETE'], koji.BUILD_STATES['FAILED']] - or c.reused_component_id + if (all(c.state in [koji.BUILD_STATES['COMPLETE'], + koji.BUILD_STATES['FAILED']] or c.reused_component_id for c in unbuilt_components) and builder.module_build_tag): further_work += [module_build_service.messaging.KojiRepoChange( 'start_build_batch: fake msg', builder.module_build_tag['name'])] @@ -204,6 +204,7 @@ def continue_batch_build(config, module, session, builder, components=None): session.commit() return further_work + def start_next_batch_build(config, module, session, builder, components=None): """ Tries to start the build of next batch. In case there are still unbuilt @@ -294,10 +295,10 @@ def start_next_batch_build(config, module, session, builder, components=None): # successfully built yet or isn't currently being built. unbuilt_components = components or [ c for c in module.component_builds - if (c.state != koji.BUILD_STATES['COMPLETE'] - and c.state != koji.BUILD_STATES['BUILDING'] - and c.state != koji.BUILD_STATES['FAILED'] - and c.batch == module.batch) + if (c.state != koji.BUILD_STATES['COMPLETE'] and + c.state != koji.BUILD_STATES['BUILDING'] and + c.state != koji.BUILD_STATES['FAILED'] and + c.batch == module.batch) ] # If there are no components to build, skip the batch and start building @@ -308,11 +309,12 @@ def start_next_batch_build(config, module, session, builder, components=None): return start_next_batch_build(config, module, session, builder) log.info("Starting build of next batch %d, %s" % (module.batch, - unbuilt_components)) + unbuilt_components)) return continue_batch_build( config, module, session, builder, unbuilt_components) + def pagination_metadata(p_query, request_args): """ Returns a dictionary containing metadata about the paginated query. @@ -358,6 +360,7 @@ def pagination_metadata(p_query, request_args): return pagination_data + def filter_component_builds(flask_request): """ Returns a flask_sqlalchemy.Pagination object based on the request parameters @@ -398,12 +401,13 @@ def filter_component_builds(flask_request): query = query.order_by(column) else: raise ValidationError('An invalid order_by or order_desc_by key ' - 'was supplied') + 'was supplied') page = flask_request.args.get('page', 1, type=int) per_page = flask_request.args.get('per_page', 10, type=int) return query.paginate(page, per_page, False) + def filter_module_builds(flask_request): """ Returns a flask_sqlalchemy.Pagination object based on the request parameters @@ -468,7 +472,7 @@ def filter_module_builds(flask_request): query = query.order_by(column) else: raise ValidationError('An invalid order_by or order_desc_by key ' - 'was supplied') + 'was supplied') page = flask_request.args.get('page', 1, type=int) per_page = flask_request.args.get('per_page', 10, type=int) @@ -567,6 +571,7 @@ def _scm_get_latest(pkg): 'error': None } + def format_mmd(mmd, scmurl): """ Prepares the modulemd for the MBS. This does things such as replacing the @@ -667,6 +672,7 @@ def format_mmd(mmd, scmurl): if err_msg: raise UnprocessableEntity(err_msg) + def merge_included_mmd(mmd, included_mmd): """ Merges two modulemds. This merges only metadata which are needed in @@ -678,6 +684,7 @@ def merge_included_mmd(mmd, included_mmd): else: mmd.xmd['mbs']['rpms'].update(included_mmd.xmd['mbs']['rpms']) + def record_component_builds(mmd, module, initial_batch=1, previous_buildorder=None, main_mmd=None): import koji # Placed here to avoid py2/py3 conflicts... @@ -788,6 +795,7 @@ def submit_module_build_from_yaml(username, handle, optional_params=None): _url_check_re = re.compile(r"^[^:/]+:.*$") + def submit_module_build_from_scm(username, url, branch, allow_local_url=False, skiptests=False, optional_params=None): # Translate local paths into file:// URL @@ -805,7 +813,6 @@ def submit_module_build_from_scm(username, url, branch, allow_local_url=False, def submit_module_build(username, url, mmd, scm, yaml, optional_params=None): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. - import module_build_service.scm module = models.ModuleBuild.query.filter_by( name=mmd.name, stream=mmd.stream, version=str(mmd.version)).first() @@ -816,8 +823,8 @@ def submit_module_build(username, url, mmd, scm, yaml, optional_params=None): if module.state not in (models.BUILD_STATES['failed'], models.BUILD_STATES['init']): err_msg = ('Module (state=%s) already exists. ' - 'Only new build or resubmission of build in "init" or ' - '"failed" state is allowed.' % module.state) + 'Only new build or resubmission of build in "init" or ' + '"failed" state is allowed.' % module.state) log.error(err_msg) raise Conflict(err_msg) log.debug('Resuming existing module build %r' % module) @@ -870,9 +877,9 @@ def scm_url_schemes(terse=False): """ scm_types = { - "git": ("git://", "git+http://", "git+https://", - "git+rsync://", "http://", "https://", "file://") - } + "git": ("git://", "git+http://", "git+https://", + "git+rsync://", "http://", "https://", "file://") + } if not terse: return scm_types @@ -882,12 +889,14 @@ def scm_url_schemes(terse=False): scheme_list.extend([scheme[:-3] for scheme in scm_schemes]) return list(set(scheme_list)) + def get_scm_url_re(): schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True))) return re.compile( r"(?P(?:(?P(" + schemes_re + r"))://(?P[^/]+))?" r"(?P/[^\?]+))\?(?P[^#]*)#(?P.+)") + def module_build_state_from_msg(msg): state = int(msg.module_build_state) # TODO better handling @@ -896,6 +905,7 @@ def module_build_state_from_msg(msg): % (state, type(state), list(models.BUILD_STATES.values()))) return state + def reuse_component(component, previous_component_build, change_state_now=False): """ @@ -943,6 +953,7 @@ def reuse_component(component, previous_component_build, ) ] + def attempt_to_reuse_all_components(builder, session, module): """ Tries to reuse all the components in a build. The components are also @@ -986,6 +997,7 @@ def attempt_to_reuse_all_components(builder, session, module): return True + def get_reusable_component(session, module, component_name): """ Returns the component (RPM) build of a module that can be reused @@ -1221,6 +1233,7 @@ def validate_koji_tag(tag_arg_names, pre='', post='-', dict_key='name'): return validation_decorator + def get_rpm_release_from_mmd(mmd): """ Returns the dist tag based on the modulemd metadata and MBS configuration. diff --git a/module_build_service/views.py b/module_build_service/views.py index eea1c600..389b9526 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -36,7 +36,7 @@ from module_build_service import models, db from module_build_service.utils import ( pagination_metadata, filter_module_builds, filter_component_builds, submit_module_build_from_scm, submit_module_build_from_yaml, - scm_url_schemes, get_scm_url_re) + get_scm_url_re) from module_build_service.errors import ( ValidationError, Forbidden, NotFound) @@ -69,6 +69,7 @@ api_v1 = { }, } + class ComponentBuildAPI(MethodView): def get(self, id): @@ -206,8 +207,8 @@ class BaseHandler(object): return {k: v for k, v in self.data.items() if k not in ["owner", "scmurl", "branch"]} def validate_optional_params(self): - forbidden_params = [k for k in self.data if k not in models.ModuleBuild.__table__.columns - and k not in ["branch"]] + forbidden_params = [k for k in self.data if k not in models.ModuleBuild.__table__.columns and + k not in ["branch"]] if forbidden_params: raise ValidationError('The request contains unspecified parameters: {}'.format(", ".join(forbidden_params))) @@ -289,11 +290,11 @@ def register_api_v1(): app.add_url_rule(val['url'], endpoint=key, view_func=module_view, - **val['options']) + **val['options']) else: app.add_url_rule(val['url'], endpoint=key, view_func=component_view, **val['options']) - + register_api_v1() diff --git a/tests/__init__.py b/tests/__init__.py index 3e94dfb2..3f18ea5e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -21,7 +21,6 @@ # Written by Matt Prahl -from nose.tools import raises, eq_ - +from nose.tools import eq_ import unittest import mock @@ -42,10 +41,10 @@ class TestAuthModule(unittest.TestCase): request.cookies.return_value = {} with self.assertRaises(module_build_service.errors.Unauthorized) as cm: - result = module_build_service.auth.get_user(request) + module_build_service.auth.get_user(request) self.assertEquals(str(cm.exception), - "No 'authorization' header found.") + "No 'authorization' header found.") @patch('module_build_service.auth._get_token_info') @patch('module_build_service.auth._get_user_info') @@ -57,10 +56,10 @@ class TestAuthModule(unittest.TestCase): # https://www.youtube.com/watch?v=G-LtddOgUCE name = "Joey Jo Jo Junior Shabadoo" mocked_get_token_info = {"active": False, "username": name, - "scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"} + "scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"} get_token_info.return_value = mocked_get_token_info - get_user_info.return_value = {"groups":["group"]} + get_user_info.return_value = {"groups": ["group"]} headers = {"authorization": "Bearer foobar"} request = mock.MagicMock() @@ -70,10 +69,10 @@ class TestAuthModule(unittest.TestCase): request.headers.__contains__.side_effect = headers.__contains__ with self.assertRaises(module_build_service.errors.Unauthorized) as cm: - result = module_build_service.auth.get_user(request) + module_build_service.auth.get_user(request) self.assertEquals(str(cm.exception), - "OIDC token invalid or expired.") + "OIDC token invalid or expired.") @patch('module_build_service.auth._get_token_info') @patch('module_build_service.auth._get_user_info') @@ -85,10 +84,10 @@ class TestAuthModule(unittest.TestCase): # https://www.youtube.com/watch?v=G-LtddOgUCE name = "Joey Jo Jo Junior Shabadoo" mocked_get_token_info = {"active": True, "username": name, - "scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"} + "scope": "openid https://id.fedoraproject.org/scope/groups mbs-scope"} get_token_info.return_value = mocked_get_token_info - get_user_info.return_value = {"groups":["group"]} + get_user_info.return_value = {"groups": ["group"]} headers = {"authorization": "Bearer foobar"} request = mock.MagicMock() @@ -115,7 +114,7 @@ class TestAuthModule(unittest.TestCase): module_build_service.auth.get_user(request) self.assertEquals(str(cm.exception), - "OIDC_CLIENT_SECRETS must be set in server config.") + "OIDC_CLIENT_SECRETS must be set in server config.") @patch('module_build_service.auth._get_token_info') @patch('module_build_service.auth._get_user_info') @@ -126,11 +125,12 @@ class TestAuthModule(unittest.TestCase): 'OIDC_REQUIRED_SCOPE': 'mbs-scope'}): # https://www.youtube.com/watch?v=G-LtddOgUCE name = "Joey Jo Jo Junior Shabadoo" - mocked_get_token_info = {"active": True, "username": name, - "scope": "openid https://id.fedoraproject.org/scope/groups"} + mocked_get_token_info = {"active": True, + "username": name, + "scope": "openid https://id.fedoraproject.org/scope/groups"} get_token_info.return_value = mocked_get_token_info - get_user_info.return_value = {"groups":["group"]} + get_user_info.return_value = {"groups": ["group"]} headers = {"authorization": "Bearer foobar"} request = mock.MagicMock() @@ -140,11 +140,11 @@ class TestAuthModule(unittest.TestCase): request.headers.__contains__.side_effect = headers.__contains__ with self.assertRaises(module_build_service.errors.Unauthorized) as cm: - result = module_build_service.auth.get_user(request) + module_build_service.auth.get_user(request) self.assertEquals(str(cm.exception), - "Required OIDC scope 'mbs-scope' not present: " - "['openid', 'https://id.fedoraproject.org/scope/groups']") + "Required OIDC scope 'mbs-scope' not present: " + "['openid', 'https://id.fedoraproject.org/scope/groups']") @patch('module_build_service.auth._get_token_info') @patch('module_build_service.auth._get_user_info') @@ -154,11 +154,12 @@ class TestAuthModule(unittest.TestCase): with patch.dict('module_build_service.app.config', {'OIDC_CLIENT_SECRETS': client_secrets}): # https://www.youtube.com/watch?v=G-LtddOgUCE name = "Joey Jo Jo Junior Shabadoo" - mocked_get_token_info = {"active": True, "username": name, - "scope": "openid https://id.fedoraproject.org/scope/groups"} + mocked_get_token_info = {"active": True, + "username": name, + "scope": "openid https://id.fedoraproject.org/scope/groups"} get_token_info.return_value = mocked_get_token_info - get_user_info.return_value = {"groups":["group"]} + get_user_info.return_value = {"groups": ["group"]} headers = {"authorization": "Bearer foobar"} request = mock.MagicMock() @@ -168,7 +169,7 @@ class TestAuthModule(unittest.TestCase): request.headers.__contains__.side_effect = headers.__contains__ with self.assertRaises(module_build_service.errors.Forbidden) as cm: - result = module_build_service.auth.get_user(request) + module_build_service.auth.get_user(request) self.assertEquals(str(cm.exception), - "OIDC_REQUIRED_SCOPE must be set in server config.") + "OIDC_REQUIRED_SCOPE must be set in server config.") diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 050c7722..802a8e42 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -24,7 +24,7 @@ import unittest import koji import vcr import os -from os import path, mkdir, remove +from os import path, mkdir from os.path import dirname from shutil import copyfile @@ -38,7 +38,6 @@ from module_build_service import db, models, conf, build_logs from mock import patch, PropertyMock from tests import app, init_data, test_reuse_component_init_data -import os import json import itertools @@ -51,6 +50,7 @@ cassette_dir = base_dir + '/vcr-request-data/' user = ('Homer J. Simpson', set(['packager'])) + class MockedSCM(object): def __init__(self, mocked_scm, name, mmd_filename, commit=None): self.mocked_scm = mocked_scm @@ -70,13 +70,14 @@ class MockedSCM(object): mkdir(scm_dir) base_dir = path.abspath(path.dirname(__file__)) copyfile(path.join(base_dir, '..', 'staged_data', self.mmd_filename), - path.join(scm_dir, self.mmd_filename)) + path.join(scm_dir, self.mmd_filename)) return scm_dir def get_latest(self, branch='master'): return branch + class TestModuleBuilder(GenericBuilder): """ Test module builder which succeeds for every build. @@ -173,7 +174,7 @@ class TestModuleBuilder(GenericBuilder): module_build_service.scheduler.consumer.work_queue_put(msg) def build(self, artifact_name, source): - print "Starting building artifact %s: %s" % (artifact_name, source) + print("Starting building artifact %s: %s" % (artifact_name, source)) TestModuleBuilder._build_id += 1 @@ -207,18 +208,18 @@ class TestModuleBuilder(GenericBuilder): @patch("module_build_service.config.Config.system", - new_callable=PropertyMock, return_value="test") + new_callable=PropertyMock, return_value="test") @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={ - 'srpm-build': - set(['shadow-utils', 'fedora-release', 'redhat-rpm-config', - 'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']), - 'build': - set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk', - 'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash', - 'info', 'bzip2', 'grep', 'redhat-rpm-config', - 'diffutils', 'make', 'patch', 'shadow-utils', - 'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])}) + 'srpm-build': + set(['shadow-utils', 'fedora-release', 'redhat-rpm-config', + 'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']), + 'build': + set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk', + 'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash', + 'info', 'bzip2', 'grep', 'redhat-rpm-config', + 'diffutils', 'make', 'patch', 'shadow-utils', + 'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])}) class TestBuild(unittest.TestCase): # Global variable used for tests if needed @@ -300,7 +301,7 @@ class TestBuild(unittest.TestCase): # or "ready" state. for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) - self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]]) # All components has to be tagged, so tag_groups and buildroot_groups are empty... self.assertEqual(tag_groups, []) @@ -323,13 +324,13 @@ class TestBuild(unittest.TestCase): return json.loads(rv.data) with patch("module_build_service.config.Config.yaml_submit_allowed", - new_callable=PropertyMock, return_value=True): + new_callable=PropertyMock, return_value=True): conf.set_item("yaml_submit_allowed", True) data = submit() self.assertEqual(data['id'], 1) with patch("module_build_service.config.Config.yaml_submit_allowed", - new_callable=PropertyMock, return_value=False): + new_callable=PropertyMock, return_value=False): data = submit() self.assertEqual(data['status'], 403) self.assertEqual(data['message'], 'YAML submission is not enabled') @@ -373,11 +374,11 @@ class TestBuild(unittest.TestCase): # method. We just cancel the build here using the web API to simulate # user cancelling the build in the middle of building. def on_build_cb(cls, artifact_name, source): - self.client.patch( - '/module-build-service/1/module-builds/' + str(module_build_id), - data=json.dumps({'state': 'failed'})) + self.client.patch('/module-build-service/1/module-builds/' + str(module_build_id), + data=json.dumps({'state': 'failed'})) cancelled_tasks = [] + def on_cancel_cb(cls, task_id): cancelled_tasks.append(task_id) @@ -433,7 +434,7 @@ class TestBuild(unittest.TestCase): # or "ready" state. for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) - self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]]) @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @@ -465,7 +466,7 @@ class TestBuild(unittest.TestCase): main_stop = module_build_service.scheduler.make_simple_stop_condition(db.session) over_threshold = conf.num_concurrent_builds < \ db.session.query(models.ComponentBuild).filter_by( - state=koji.BUILD_STATES['BUILDING']).count() + state=koji.BUILD_STATES['BUILDING']).count() return main_stop(message) or over_threshold msgs = [] @@ -477,7 +478,7 @@ class TestBuild(unittest.TestCase): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) # When this fails, it can mean that num_concurrent_builds # threshold has been met. - self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]]) @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @@ -600,7 +601,7 @@ class TestBuild(unittest.TestCase): @patch("module_build_service.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=1) def test_all_builds_in_batch_fail(self, conf_num_concurrent_builds, mocked_scm, - mocked_get_user, conf_system, dbg): + mocked_get_user, conf_system, dbg): """ Tests that if the build in batch fails, other components in a batch are still build, but next batch is not started. @@ -661,8 +662,8 @@ class TestBuild(unittest.TestCase): tag_groups = [] tag_groups.append(set( ['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357', - 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', - 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) + 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', + 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) def on_tag_artifacts_cb(cls, artifacts): self.assertEqual(tag_groups.pop(0), set(artifacts)) @@ -671,8 +672,9 @@ class TestBuild(unittest.TestCase): buildtag_groups = [] buildtag_groups.append(set( ['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357', - 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', - 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) + 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', + 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) + def on_buildroot_add_artifacts_cb(cls, artifacts, install): self.assertEqual(buildtag_groups.pop(0), set(artifacts)) TestModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb @@ -697,7 +699,7 @@ class TestBuild(unittest.TestCase): @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') def test_submit_build_reuse_all_without_build_macros(self, mocked_scm, mocked_get_user, - conf_system, dbg): + conf_system, dbg): """ Tests that we can reuse components even when the reused module does not have module-build-macros component. @@ -718,8 +720,8 @@ class TestBuild(unittest.TestCase): tag_groups = [] tag_groups.append(set( ['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357', - 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', - 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) + 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', + 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) def on_tag_artifacts_cb(cls, artifacts): self.assertEqual(tag_groups.pop(0), set(artifacts)) @@ -728,8 +730,9 @@ class TestBuild(unittest.TestCase): buildtag_groups = [] buildtag_groups.append(set( ['perl-Tangerine-0.23-1.module_testmodule_master_20170109091357', - 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', - 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) + 'perl-List-Compare-0.53-5.module_testmodule_master_20170109091357', + 'tangerine-0.22-3.module_testmodule_master_20170109091357'])) + def on_buildroot_add_artifacts_cb(cls, artifacts, install): self.assertEqual(buildtag_groups.pop(0), set(artifacts)) TestModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb @@ -742,7 +745,7 @@ class TestBuild(unittest.TestCase): # or "ready" state. for build in models.ComponentBuild.query.filter_by(module_id=2).all(): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) - self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]]) self.assertNotEqual(build.package, "module-build-macros") @timed(60) @@ -769,7 +772,7 @@ class TestBuild(unittest.TestCase): # Set the components from batch 2 to COMPLETE components = models.ComponentBuild.query.filter_by(module_id=module_build_id) for c in components: - print c + print(c) if c.batch == 2: c.state = koji.BUILD_STATES["COMPLETE"] db.session.commit() @@ -783,4 +786,4 @@ class TestBuild(unittest.TestCase): # or "ready" state. for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) - self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]]) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index e107faed..762e4d89 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -36,6 +36,7 @@ from tests import conf, init_data from module_build_service.builder import KojiModuleBuilder + class FakeKojiModuleBuilder(KojiModuleBuilder): @module_build_service.utils.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError)) @@ -50,6 +51,7 @@ class FakeKojiModuleBuilder(KojiModuleBuilder): return koji_session + class TestKojiBuilder(unittest.TestCase): def setUp(self): @@ -234,4 +236,3 @@ class TestGetKojiClientSession(unittest.TestCase): components=[]) args, kwargs = mocked_krb_login.call_args self.assertTrue(set([('proxyuser', self.module.owner)]).issubset(set(kwargs.items()))) - diff --git a/tests/test_config.py b/tests/test_config.py index 0787293d..4e3c8691 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -20,16 +20,12 @@ # # Written by Jan Kaluza -from nose.tools import raises, eq_ - - import unittest -import mock import os.path -from mock import patch from module_build_service import conf + class TestConfig(unittest.TestCase): def test_path_expanduser(self): test_dir = "~/modulebuild/builds" diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index 79b89606..c811c596 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -28,10 +28,8 @@ import os from os import path from os.path import dirname - import module_build_service.messaging import module_build_service.scheduler.handlers.repos -import module_build_service.utils from module_build_service import models, conf, build_logs from mock import patch, Mock @@ -60,7 +58,6 @@ class TestBuild(unittest.TestCase): self.vcr = vcr.use_cassette(filename) self.vcr.__enter__() - def tearDown(self): # Necessary to restart the twisted reactor for the next test. import sys @@ -88,8 +85,8 @@ class TestBuild(unittest.TestCase): pkg_res.return_value = Mock() pkg_res.return_value.version = "current-tested-version" rpm_mock = Mock() - rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \ - "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2" + rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \ + "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2" attrs = {'communicate.return_value': (rpm_out, 'error'), 'wait.return_value': 0} rpm_mock.configure_mock(**attrs) @@ -127,8 +124,8 @@ class TestBuild(unittest.TestCase): pkg_res.return_value = Mock() pkg_res.return_value.version = "current-tested-version" rpm_mock = Mock() - rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \ - "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2" + rpm_out = "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \ + "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2" attrs = {'communicate.return_value': (rpm_out, 'error'), 'wait.return_value': 0} rpm_mock.configure_mock(**attrs) @@ -149,7 +146,6 @@ class TestBuild(unittest.TestCase): rpms_in_tag.assert_called_once() self.assertEqual(expected_output, ret) - def test_prepare_file_directory(self): """ Test preparation of directory with output files """ dir_path = self.cg._prepare_file_directory() diff --git a/tests/test_logger.py b/tests/test_logger.py index a0156fe7..86be04d6 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -29,7 +29,6 @@ import unittest from module_build_service import log from module_build_service.logger import ModuleBuildLogs from module_build_service.scheduler.consumer import MBSConsumer -from mock import patch, PropertyMock class TestLogger(unittest.TestCase): diff --git a/tests/test_messaging.py b/tests/test_messaging.py index 42891b06..042e2408 100644 --- a/tests/test_messaging.py +++ b/tests/test_messaging.py @@ -22,7 +22,7 @@ import unittest -from module_build_service import messaging, conf +from module_build_service import messaging from mock import patch, PropertyMock @@ -55,7 +55,7 @@ class TestFedmsgMessaging(unittest.TestCase): self.assertEqual(msg.build_new_state, 1) @patch("module_build_service.config.Config.system", - new_callable=PropertyMock, return_value="copr") + new_callable=PropertyMock, return_value="copr") def test_copr_build_end(self, conf_system): # http://fedora-fedmsg.readthedocs.io/en/latest/topics.html#copr-build-end copr_build_end_msg = { @@ -116,7 +116,6 @@ class TestFedmsgMessaging(unittest.TestCase): self.assertEqual(msg.tag, "module-debugging-tools-master-20170405115403-build") self.assertEqual(msg.artifact, "module-build-macros") - def test_buildsys_repo_done(self): # https://fedora-fedmsg.readthedocs.io/en/latest/topics.html#id134 buildsys_tag_msg = { diff --git a/tests/test_pdc.py b/tests/test_pdc.py index 5c151248..d0826bcf 100644 --- a/tests/test_pdc.py +++ b/tests/test_pdc.py @@ -34,6 +34,7 @@ import modulemd base_dir = os.path.dirname(__file__) cassette_dir = base_dir + '/vcr-request-data/' + class TestPDCModule(unittest.TestCase): def setUp(self): @@ -111,5 +112,5 @@ class TestPDCModule(unittest.TestCase): set(['shadow-utils', 'redhat-rpm-config', 'rpm-build', 'fedora-modular-release', 'fedpkg-minimal', 'gnupg2', 'bash']) - } + } self.assertEqual(result, expected) diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index 447d65a3..eb0bedb0 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -30,7 +30,7 @@ import os import vcr import koji from tests import conf, db, app, scheduler_init_data -from module_build_service import conf, build_logs +from module_build_service import build_logs base_dir = os.path.dirname(os.path.dirname(__file__)) cassette_dir = base_dir + '/vcr-request-data/' @@ -85,11 +85,11 @@ class TestModuleWait(unittest.TestCase): from_module_event.return_value = mocked_module_build msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1, - module_build_state='some state') + module_build_state='some state') self.fn(config=self.config, session=self.session, msg=msg) @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", - return_value={'build': [], 'srpm-build': []}) + return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.builder.KojiModuleBuilder.get_session") @patch("module_build_service.builder.GenericBuilder.create_from_module") @patch('module_build_service.pdc') @@ -113,13 +113,13 @@ class TestModuleWait(unittest.TestCase): create_builder.return_value = builder msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1, - module_build_state='some state') + module_build_state='some state') module_build_service.scheduler.handlers.modules.wait( config=conf, session=db.session, msg=msg) koji_session.newRepo.assert_called_once_with("module-123-build") @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", - return_value={'build': [], 'srpm-build': []}) + return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.builder.KojiModuleBuilder.get_session") @patch("module_build_service.builder.GenericBuilder.create_from_module") @patch('module_build_service.pdc') @@ -143,7 +143,7 @@ class TestModuleWait(unittest.TestCase): create_builder.return_value = builder msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1, - module_build_state='some state') + module_build_state='some state') module_build_service.scheduler.handlers.modules.wait( config=conf, session=db.session, msg=msg) self.assertTrue(not koji_session.newRepo.called) diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index f9336575..b8383b58 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -20,19 +20,11 @@ import unittest from os import path -import vcr -import modulemd from mock import patch -import module_build_service.utils -import module_build_service.scm from module_build_service import models, conf -from module_build_service.errors import ProgrammingError, ValidationError from tests import test_reuse_component_init_data, init_data, db import mock -from mock import PropertyMock import koji -import module_build_service.scheduler.handlers.components -from module_build_service.builder import GenericBuilder, KojiModuleBuilder from module_build_service.scheduler.producer import MBSProducer import six.moves.queue as queue from datetime import datetime, timedelta @@ -41,6 +33,7 @@ BASE_DIR = path.abspath(path.dirname(__file__)) CASSETTES_DIR = path.join( path.abspath(path.dirname(__file__)), '..', 'vcr-request-data') + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.scheduler.consumer.get_global_consumer") @@ -123,10 +116,9 @@ class TestPoller(unittest.TestCase): koji_session.newRepo.assert_called_once_with("module-testmodule-build") - def test_trigger_new_repo_when_succeded(self, create_builder, - koji_get_session, global_consumer, - dbg): + koji_get_session, global_consumer, + dbg): """ Tests that we do not call koji_sesion.newRepo when newRepo task succeeded. diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py index a6df0fe0..5259d993 100644 --- a/tests/test_scheduler/test_repo_done.py +++ b/tests/test_scheduler/test_repo_done.py @@ -34,6 +34,7 @@ from tests import conf, db, app base_dir = dirname(dirname(__file__)) cassette_dir = base_dir + '/vcr-request-data/' + class TestRepoDone(unittest.TestCase): def setUp(self): diff --git a/tests/test_scheduler/test_tag_tagged.py b/tests/test_scheduler/test_tag_tagged.py index bf0ce4c7..f5abe7ba 100644 --- a/tests/test_scheduler/test_tag_tagged.py +++ b/tests/test_scheduler/test_tag_tagged.py @@ -20,10 +20,8 @@ # # Written by Jan Kaluza -from os.path import dirname import unittest import mock -import vcr from mock import patch @@ -31,10 +29,11 @@ import module_build_service.messaging import module_build_service.scheduler.handlers.repos import module_build_service.models from tests import test_reuse_component_init_data -from tests import conf, db, app +from tests import conf, db import koji + class TestTagTagged(unittest.TestCase): def setUp(self): @@ -63,9 +62,8 @@ class TestTagTagged(unittest.TestCase): module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) - @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", - return_value={'build': [], 'srpm-build': []}) + return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.builder.KojiModuleBuilder.get_session") @patch("module_build_service.builder.GenericBuilder.create_from_module") def test_newrepo(self, create_builder, koji_get_session, dbg): @@ -123,9 +121,8 @@ class TestTagTagged(unittest.TestCase): # status later in poller. self.assertEqual(module_build.new_repo_task_id, 123456) - @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", - return_value={'build': [], 'srpm-build': []}) + return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.builder.KojiModuleBuilder.get_session") @patch("module_build_service.builder.GenericBuilder.create_from_module") def test_newrepo_still_building_components(self, create_builder, koji_get_session, dbg): @@ -161,7 +158,7 @@ class TestTagTagged(unittest.TestCase): self.assertTrue(not koji_session.newRepo.called) @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", - return_value={'build': [], 'srpm-build': []}) + return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.builder.KojiModuleBuilder.get_session") @patch("module_build_service.builder.GenericBuilder.create_from_module") def test_newrepo_failed_components(self, create_builder, koji_get_session, dbg): @@ -215,7 +212,7 @@ class TestTagTagged(unittest.TestCase): self.assertEqual(module_build.new_repo_task_id, 123456) @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", - return_value = {'build': [], 'srpm-build': []}) + return_value={'build': [], 'srpm-build': []}) @patch("module_build_service.builder.KojiModuleBuilder.get_session") @patch("module_build_service.builder.GenericBuilder.create_from_module") def test_newrepo_multiple_batches_tagged( diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 13dd1710..8c38104a 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -31,7 +31,6 @@ from module_build_service.errors import ProgrammingError, ValidationError from tests import (test_reuse_component_init_data, init_data, db, test_reuse_shared_userspace_init_data) import mock -from mock import PropertyMock import koji import module_build_service.scheduler.handlers.components from module_build_service.builder import GenericBuilder, KojiModuleBuilder @@ -41,6 +40,7 @@ BASE_DIR = path.abspath(path.dirname(__file__)) CASSETTES_DIR = path.join( path.abspath(path.dirname(__file__)), '..', 'vcr-request-data') + class MockedSCM(object): def __init__(self, mocked_scm, name, mmd_filename, commit=None): self.mocked_scm = mocked_scm @@ -60,13 +60,14 @@ class MockedSCM(object): mkdir(scm_dir) base_dir = path.abspath(path.dirname(__file__)) copyfile(path.join(base_dir, '..', 'staged_data', self.mmd_filename), - path.join(scm_dir, self.name + ".yaml")) + path.join(scm_dir, self.name + ".yaml")) return scm_dir def get_latest(self, branch='master'): return self.commit if self.commit else branch + class TestUtils(unittest.TestCase): def setUp(self): @@ -115,8 +116,8 @@ class TestUtils(unittest.TestCase): 'stream': 'master', 'version': '20170404161234'}}, 'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'}, - 'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'}, - 'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}}, + 'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'}, + 'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}}, 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/testmodule' '.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4', } @@ -133,6 +134,7 @@ class TestUtils(unittest.TestCase): 'f24': '4ceea43add2366d8b8c5a622a2fb563b625b9abf', 'f23': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c', 'f25': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'} + def mocked_get_latest(branch="master"): return hashes_returned[branch] mocked_scm.return_value.get_latest = mocked_get_latest @@ -152,8 +154,8 @@ class TestUtils(unittest.TestCase): 'stream': 'master', 'version': '20170404161234'}}, 'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'}, - 'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'}, - 'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}}, + 'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'}, + 'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}}, 'scmurl': None, } } @@ -368,7 +370,7 @@ class TestUtils(unittest.TestCase): self.assertEquals( validate_koji_tag_good_tag_value_in_dict_nondefault_key( - {'nondefault': 'module-foo'}), True) + {'nondefault': 'module-foo'}), True) def test_validate_koji_tag_double_trouble_good(self): """ Test that we pass on a list of tags that are good. """ @@ -413,7 +415,7 @@ class TestUtils(unittest.TestCase): component states properly. """ MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', - '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') with app.app_context(): test_reuse_component_init_data() # Mark the module build as failed, so we can resubmit it. @@ -502,7 +504,6 @@ class DummyModuleBuilder(GenericBuilder): self.tag_name = tag_name self.config = config - def buildroot_connect(self, groups): pass @@ -548,6 +549,7 @@ class DummyModuleBuilder(GenericBuilder): def list_tasks_for_components(self, component_builds=None, state='active'): pass + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []}) class TestBatches(unittest.TestCase): @@ -639,9 +641,6 @@ class TestBatches(unittest.TestCase): builder = mock.MagicMock() builder.buildroot_ready.return_value = False - further_work = module_build_service.utils.start_next_batch_build( - conf, module_build, db.session, builder) # Batch number should not increase. self.assertEqual(module_build.batch, 1) - diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index a6053710..1056bab1 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -28,7 +28,7 @@ import vcr import modulemd as _modulemd import module_build_service.scm -from mock import patch, Mock, PropertyMock, MagicMock +from mock import patch, PropertyMock, MagicMock from shutil import copyfile from os import path, mkdir from os.path import dirname @@ -96,7 +96,7 @@ class MockedSCM(object): mkdir(scm_dir) base_dir = path.abspath(path.dirname(__file__)) copyfile(path.join(base_dir, '..', 'staged_data', mmd_filename), - path.join(scm_dir, self.name + ".yaml")) + path.join(scm_dir, self.name + ".yaml")) self.checkout_id += 1 @@ -258,7 +258,7 @@ class TestViews(unittest.TestCase): def test_query_builds_filter_task_id(self): rv = self.client.get('/module-build-service/1/component-builds/?task_id=12312346') data = json.loads(rv.data) - self.assertEquals(data['meta']['total'], 1) + self.assertEquals(data['meta']['total'], 1) def test_query_builds_filter_name(self): rv = self.client.get('/module-build-service/1/module-builds/?name=nginx') @@ -335,7 +335,7 @@ class TestViews(unittest.TestCase): def test_query_builds_order_by(self): rv = self.client.get('/module-build-service/1/module-builds/?' - 'per_page=10&order_by=id') + 'per_page=10&order_by=id') items = json.loads(rv.data)['items'] # Check that the id is 1, 2, 3, ..., 10 for idx, item in enumerate(items): @@ -343,7 +343,7 @@ class TestViews(unittest.TestCase): def test_query_builds_order_desc_by(self): rv = self.client.get('/module-build-service/1/module-builds/?' - 'per_page=10&order_desc_by=id') + 'per_page=10&order_desc_by=id') items = json.loads(rv.data)['items'] # Check that the id is items[0]["id"], items[0]["id"] - 1, ... for idx, item in enumerate(items): @@ -355,7 +355,7 @@ class TestViews(unittest.TestCase): we prefer order_desc_by. """ rv = self.client.get('/module-build-service/1/module-builds/?' - 'per_page=10&order_desc_by=id&order_by=name') + 'per_page=10&order_desc_by=id&order_by=name') items = json.loads(rv.data)['items'] # Check that the id is items[0]["id"], items[0]["id"] - 1, ... for idx, item in enumerate(items): @@ -363,7 +363,7 @@ class TestViews(unittest.TestCase): def test_query_builds_order_by_wrong_key(self): rv = self.client.get('/module-build-service/1/module-builds/?' - 'per_page=10&order_by=unknown') + 'per_page=10&order_by=unknown') data = json.loads(rv.data) self.assertEquals(data['status'], 400) self.assertEquals(data['error'], 'Bad Request') @@ -472,7 +472,7 @@ class TestViews(unittest.TestCase): {'branch': 'master', 'scmurl': 'git://badurl.com'})) data = json.loads(rv.data) self.assertEquals(data['message'], 'The submitted scmurl ' - 'git://badurl.com is not allowed') + 'git://badurl.com is not allowed') self.assertEquals(data['status'], 403) self.assertEquals(data['error'], 'Forbidden') @@ -483,15 +483,15 @@ class TestViews(unittest.TestCase): 'testmodule.git'})) data = json.loads(rv.data) self.assertEquals(data['message'], 'The submitted scmurl ' - 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git ' - 'is not valid') + 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git ' + 'is not valid') self.assertEquals(data['status'], 403) self.assertEquals(data['error'], 'Forbidden') @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') def test_submit_build_bad_modulemd(self, mocked_scm, mocked_get_user): - mocked_scm_obj = MockedSCM(mocked_scm, "bad", "bad.yaml") + MockedSCM(mocked_scm, "bad", "bad.yaml") rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' @@ -562,8 +562,9 @@ class TestViews(unittest.TestCase): @patch("module_build_service.config.Config.modules_allow_repository", new_callable=PropertyMock, return_value=True) def test_submit_build_includedmodule(self, conf, mocked_scm, mocked_get_user): - mocked_scm_obj = MockedSCM(mocked_scm, "includedmodules", - ["includedmodules.yaml", "testmodule.yaml"]) + MockedSCM(mocked_scm, "includedmodules", ["includedmodules.yaml", + "testmodule.yaml"]) + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) @@ -609,9 +610,9 @@ class TestViews(unittest.TestCase): @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') def test_submit_build_includedmodule_custom_repo_not_allowed(self, - mocked_scm, mocked_get_user): - mocked_scm_obj = MockedSCM(mocked_scm, "includedmodules", - ["includedmodules.yaml", "testmodule.yaml"]) + mocked_scm, mocked_get_user): + MockedSCM(mocked_scm, "includedmodules", ["includedmodules.yaml", + "testmodule.yaml"]) rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) @@ -651,9 +652,9 @@ class TestViews(unittest.TestCase): return_value=('sammy', set(["packager", "mbs-admin"]))) def test_cancel_build_admin(self, mocked_get_user): with patch("module_build_service.config.Config.admin_groups", - new_callable=PropertyMock, return_value=set(["mbs-admin"])): + new_callable=PropertyMock, return_value=set(["mbs-admin"])): rv = self.client.patch('/module-build-service/1/module-builds/30', - data=json.dumps({'state': 'failed'})) + data=json.dumps({'state': 'failed'})) data = json.loads(rv.data) self.assertEquals(data['state'], 4) @@ -663,9 +664,9 @@ class TestViews(unittest.TestCase): return_value=('sammy', set(["packager"]))) def test_cancel_build_no_admin(self, mocked_get_user): with patch("module_build_service.config.Config.admin_groups", - new_callable=PropertyMock, return_value=set(["mbs-admin"])): + new_callable=PropertyMock, return_value=set(["mbs-admin"])): rv = self.client.patch('/module-build-service/1/module-builds/30', - data=json.dumps({'state': 'failed'})) + data=json.dumps({'state': 'failed'})) data = json.loads(rv.data) self.assertEquals(data['status'], 403) @@ -861,7 +862,7 @@ class TestViews(unittest.TestCase): def submit(scmurl): return self.client.post('/module-build-service/1/module-builds/', data=json.dumps( - {'branch': 'master', 'scmurl': scmurl})) + {'branch': 'master', 'scmurl': scmurl})) allow_custom_scmurls.return_value = False res1 = submit('git://some.custom.url.org/modules/testmodule.git?#68931c9') From 04545b449dd053f40d3efb0a3ef2a9822f1f9fb2 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Mon, 17 Jul 2017 17:46:19 +0200 Subject: [PATCH 2/2] add dogpile.cache & re-sort --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6c2823e4..d1ceea44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,22 +2,23 @@ Flask Flask-Migrate Flask-SQLAlchemy Flask-Script +dogpile.cache fedmsg -moksha.hub -psutil funcsigs # Python2 only +futures # Python 2 only httplib2 kobo m2crypto m2ext mock modulemd>=1.2.0,<2.0.0 +moksha.hub munch pdc-client +psutil pyOpenSSL python-fedora qpid-python six sqlalchemy -futures # Python 2 only tabulate