From 55687e582eb4fa20a8dae392650ad84ebd650de7 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:51:26 +0100 Subject: [PATCH 01/11] lint --- module_build_service/utils.py | 39 +++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 53c7fd66..b1d66f27 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -20,9 +20,8 @@ # # Written by Ralph Bean # Matt Prahl + """ Utility functions for module_build_service. """ -from flask import request, url_for -from datetime import datetime import re import functools import time @@ -30,13 +29,17 @@ import shutil import tempfile import os import modulemd + +from flask import request, url_for +from datetime import datetime + from module_build_service import log, models from module_build_service.errors import ValidationError, UnprocessableEntity -from module_build_service import app, conf, db, log -from module_build_service.errors import ( - ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound) +from module_build_service import conf, db +from module_build_service.errors import (Unauthorized, Conflict) from multiprocessing.dummy import Pool as ThreadPool + def retry(timeout=120, interval=30, wait_on=Exception): """ A decorator that allows to retry a section of code... ...until success or timeout. @@ -64,7 +67,7 @@ def start_next_build_batch(config, module, session, builder, components=None): import koji # Placed here to avoid py2/py3 conflicts... if any([c.state == koji.BUILD_STATES['BUILDING'] - for c in module.component_builds ]): + for c in module.component_builds]): raise ValueError("Cannot start a batch when another is in flight.") # The user can either pass in a list of components to 'seed' the batch, or @@ -148,7 +151,7 @@ def filter_module_builds(flask_request): # Filter the query based on date request parameters for item in ('submitted', 'modified', 'completed'): for context in ('before', 'after'): - request_arg = '%s_%s' % (item, context) # i.e. submitted_before + request_arg = '%s_%s' % (item, context) # i.e. submitted_before iso_datetime_arg = request.args.get(request_arg, None) if iso_datetime_arg: @@ -171,6 +174,7 @@ def filter_module_builds(flask_request): per_page = flask_request.args.get('per_page', 10, type=int) return query.paginate(page, per_page, False) + def submit_module_build(username, url): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. @@ -217,24 +221,24 @@ def submit_module_build(username, url): mmd.version = int(scm.version) module = models.ModuleBuild.query.filter_by(name=mmd.name, - stream=mmd.stream, - version=mmd.version).first() + stream=mmd.stream, + version=mmd.version).first() if module: log.debug('Checking whether module build already exist.') - # TODO: make this configurable, we might want to allow - # resubmitting any stuck build on DEV no matter the state + # TODO: make this configurable, we might want to allow + # resubmitting any stuck build on DEV no matter the state if module.state not in (models.BUILD_STATES['failed'],): log.error('Module (state=%s) already exists. ' - 'Only new or failed builds are allowed.' - % module.state) + 'Only new or failed builds are allowed.' + % module.state) raise Conflict('Module (state=%s) already exists. ' - 'Only new or failed builds are allowed.' - % module.state) + 'Only new or failed builds are allowed.' + % module.state) log.debug('Resuming existing module build %r' % module) module.username = username module.transition(conf, models.BUILD_STATES["init"]) log.info("Resumed existing module build in previous state %s" - % module.state) + % module.state) else: log.debug('Creating new module build') module = models.ModuleBuild.create( @@ -296,8 +300,7 @@ def submit_module_build(username, url): existing_build = models.ComponentBuild.query.filter_by( module_id=module.id, package=pkgname).first() - if (existing_build - and existing_build.state != models.BUILD_STATES['done']): + if (existing_build and existing_build.state != models.BUILD_STATES['done']): existing_build.state = models.BUILD_STATES['init'] db.session.add(existing_build) else: From 12973804c8060cab1de2ab16463f94f7e88d1a3a Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:52:09 +0100 Subject: [PATCH 02/11] Network timeouts + lint --- config.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index d5e27caf..283b7d7c 100644 --- a/config.py +++ b/config.py @@ -12,8 +12,12 @@ class BaseConfiguration(object): HOST = '127.0.0.1' PORT = 5000 + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + SYSTEM = 'koji' - MESSAGING = 'fedmsg' # or amq + MESSAGING = 'fedmsg' # or amq KOJI_CONFIG = '/etc/module_build_service/koji.conf' KOJI_PROFILE = 'koji' KOJI_ARCHES = ['i686', 'armv7hl', 'x86_64'] @@ -60,18 +64,22 @@ class BaseConfiguration(object): # AMQ prefixed variables are required only while using 'amq' as messaging backend # Addresses to listen to AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji', - 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service',] + 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service'] # Address for sending messages AMQ_DEST_ADDRESS = 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service' AMQ_CERT_FILE = '/etc/module_build_service/msg-m8y-client.crt' AMQ_PRIVATE_KEY_FILE = '/etc/module_build_service/msg-m8y-client.key' AMQ_TRUSTED_CERT_FILE = '/etc/module_build_service/Root-CA.crt' + class DevConfiguration(BaseConfiguration): LOG_BACKEND = 'console' LOG_LEVEL = 'debug' HOST = '0.0.0.0' + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 if path.exists('/home/fedora/modularity.keytab'): KRB_PRINCIPAL = 'modularity@STG.FEDORAPROJECT.ORG' @@ -86,20 +94,25 @@ class DevConfiguration(BaseConfiguration): REQUIRE_PACKAGER = False # You only need these FAS options if you turn on authorization # with REQUIRE_PACKAGER=True - #FAS_USERNAME = 'put your fas username here' - #FAS_PASSWORD = 'put your fas password here....' - #FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here - #FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip() + # FAS_USERNAME = 'put your fas username here' + # FAS_PASSWORD = 'put your fas password here....' + # FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here + # FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip() KOJI_ARCHES = ['x86_64'] + class TestConfiguration(BaseConfiguration): LOG_BACKEND = 'console' LOG_LEVEL = 'debug' SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:' DEBUG = True + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 + class ProdConfiguration(BaseConfiguration): FAS_USERNAME = 'TODO' - #FAS_PASSWORD = 'another password' + # FAS_PASSWORD = 'another password' From 1cc321a4406e4e430e5bf1756e6ff89953d40033 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:52:23 +0100 Subject: [PATCH 03/11] Default network timeouts --- module_build_service/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module_build_service/config.py b/module_build_service/config.py index f6937dbf..c35943ec 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -200,6 +200,14 @@ class Config(object): 'type': list, 'default': [], 'desc': 'Allowed SCM URLs.'}, + 'net_timeout': { + 'type': int, + 'default': 120, + 'desc': 'Global network timeout for read/write operations, in seconds.'}, + 'net_retry_interval': { + 'type': int, + 'default': 30, + 'desc': 'Global network retry interval for read/write operations, in seconds.'}, } def __init__(self): From 47ccf6d318f4c92aa0473797990992881cdfd203 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:54:57 +0100 Subject: [PATCH 04/11] Uncomment /etc/module_build_service stuff kor koji.conf --- jenkins-check-Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-check-Dockerfile b/jenkins-check-Dockerfile index 756e51a2..debe3491 100644 --- a/jenkins-check-Dockerfile +++ b/jenkins-check-Dockerfile @@ -19,13 +19,13 @@ RUN dnf install -y \ swig \ && dnf autoremove -y \ && dnf clean all \ - && mkdir /opt/module_build_service/ - ###&& mkdir /etc/module_build_service + && mkdir /opt/module_build_service/ \ + && mkdir /etc/module_build_service WORKDIR /opt/module_build_service/ COPY ./requirements.txt /opt/module_build_service/ RUN pip install --user -r ./requirements.txt -###RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf \ +RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf ### && ln -s /opt/module_build_service/copr.conf /etc/module_build_service/copr.conf \ ### && ln -s /opt/module_build_service/krb5-stg.fp.o /etc/krb5.conf.d/stg_fedoraproject_org From f2a8801277dd7892464c2201064a895dc36f133c Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 14:00:50 +0100 Subject: [PATCH 05/11] Lint --- tests/test_builder/test_koji.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 3906433c..82bb1b46 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -28,6 +28,7 @@ import module_build_service.scheduler.handlers.repos import module_build_service.models import module_build_service.builder + class TestKojiBuilder(unittest.TestCase): def setUp(self): @@ -35,7 +36,6 @@ class TestKojiBuilder(unittest.TestCase): self.config.koji_profile = 'staging' self.config.koji_repository_url = 'https://kojipkgs.stg.fedoraproject.org/repos' - def test_tag_to_repo(self): """ Test that when a repo msg hits us and we have no match, that we do nothing gracefully. From 797a339b99eca40c8151466bdda25b5c1fd60bee Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:21:32 +0100 Subject: [PATCH 06/11] add: Testing config values for koji --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index 283b7d7c..ac88a3d2 100644 --- a/config.py +++ b/config.py @@ -112,6 +112,9 @@ class TestConfiguration(BaseConfiguration): NET_TIMEOUT = 5 NET_RETRY_INTERVAL = 1 + KOJI_PROFILE = 'staging' + KOJI_REPOSITORY_URL = 'https://kojipkgs.stg.fedoraproject.org/repos' + class ProdConfiguration(BaseConfiguration): FAS_USERNAME = 'TODO' From 371f3196c9e9c948b692030e4d754b43620404d2 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:21:56 +0100 Subject: [PATCH 07/11] Load defaults from config --- module_build_service/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index b1d66f27..5b3f2174 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -40,7 +40,7 @@ from module_build_service.errors import (Unauthorized, Conflict) from multiprocessing.dummy import Pool as ThreadPool -def retry(timeout=120, interval=30, wait_on=Exception): +def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception): """ A decorator that allows to retry a section of code... ...until success or timeout. """ From d5dfbde5d5ff96aff69b37d0d783821941146fd2 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:22:29 +0100 Subject: [PATCH 08/11] Load config --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index ac8e39b1..7533b39f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -19,11 +19,14 @@ # SOFTWARE. # # Written by Matt Prahl Date: Wed, 16 Nov 2016 15:22:48 +0100 Subject: [PATCH 09/11] PEP8 --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 7533b39f..09401879 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -39,7 +39,7 @@ def init_data(): build_one.stream = '1' build_one.version = 2 build_one.state = 3 - build_one.modulemd = '' # Skipping since no tests rely on it + build_one.modulemd = '' # Skipping since no tests rely on it build_one.koji_tag = 'module-nginx-1.2' build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?' '#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9') From 45cdd608dceaf7e13f960d962b2d33e397cc0929 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:23:40 +0100 Subject: [PATCH 10/11] Load testing values from config --- tests/test_builder/test_koji.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 82bb1b46..1b1df10c 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -28,13 +28,17 @@ import module_build_service.scheduler.handlers.repos import module_build_service.models import module_build_service.builder +from mock import patch + +from tests import conf + class TestKojiBuilder(unittest.TestCase): def setUp(self): self.config = mock.Mock() - self.config.koji_profile = 'staging' - self.config.koji_repository_url = 'https://kojipkgs.stg.fedoraproject.org/repos' + self.config.koji_profile = conf.koji_profile + self.config.koji_repository_url = conf.koji_repository_url def test_tag_to_repo(self): """ Test that when a repo msg hits us and we have no match, From e2e6639ba7622359bb9aaafaca199c73078b2d29 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:24:53 +0100 Subject: [PATCH 11/11] wip: new test for KojiModuleBuilder.buildroot_ready() --- tests/test_builder/test_koji.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 1b1df10c..f79c3b3a 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -50,3 +50,16 @@ class TestKojiBuilder(unittest.TestCase): "x86_64") self.assertEquals(repo, "https://kojipkgs.stg.fedoraproject.org/repos" "/module-base-runtime-0.25-9/latest/x86_64") + + @patch('koji.util.checkForBuilds') + def test_buildroot_ready(self, mocked_kojiutil): + + def mocked_checkForBuilds(*args, **kwargs): + raise IOError + + mocked_kojiutil.return_value.checkForBuilds = mocked_checkForBuilds + kmb = module_build_service.builder.KojiModuleBuilder(owner='Moe Szyslak', + module='nginx', + config=conf, + tag_name='module-nginx-1.2') + kmb.buildroot_ready()