From 55687e582eb4fa20a8dae392650ad84ebd650de7 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:51:26 +0100 Subject: [PATCH 01/24] lint --- module_build_service/utils.py | 39 +++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 53c7fd66..b1d66f27 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -20,9 +20,8 @@ # # Written by Ralph Bean # Matt Prahl + """ Utility functions for module_build_service. """ -from flask import request, url_for -from datetime import datetime import re import functools import time @@ -30,13 +29,17 @@ import shutil import tempfile import os import modulemd + +from flask import request, url_for +from datetime import datetime + from module_build_service import log, models from module_build_service.errors import ValidationError, UnprocessableEntity -from module_build_service import app, conf, db, log -from module_build_service.errors import ( - ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound) +from module_build_service import conf, db +from module_build_service.errors import (Unauthorized, Conflict) from multiprocessing.dummy import Pool as ThreadPool + def retry(timeout=120, interval=30, wait_on=Exception): """ A decorator that allows to retry a section of code... ...until success or timeout. @@ -64,7 +67,7 @@ def start_next_build_batch(config, module, session, builder, components=None): import koji # Placed here to avoid py2/py3 conflicts... if any([c.state == koji.BUILD_STATES['BUILDING'] - for c in module.component_builds ]): + for c in module.component_builds]): raise ValueError("Cannot start a batch when another is in flight.") # The user can either pass in a list of components to 'seed' the batch, or @@ -148,7 +151,7 @@ def filter_module_builds(flask_request): # Filter the query based on date request parameters for item in ('submitted', 'modified', 'completed'): for context in ('before', 'after'): - request_arg = '%s_%s' % (item, context) # i.e. submitted_before + request_arg = '%s_%s' % (item, context) # i.e. submitted_before iso_datetime_arg = request.args.get(request_arg, None) if iso_datetime_arg: @@ -171,6 +174,7 @@ def filter_module_builds(flask_request): per_page = flask_request.args.get('per_page', 10, type=int) return query.paginate(page, per_page, False) + def submit_module_build(username, url): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. @@ -217,24 +221,24 @@ def submit_module_build(username, url): mmd.version = int(scm.version) module = models.ModuleBuild.query.filter_by(name=mmd.name, - stream=mmd.stream, - version=mmd.version).first() + stream=mmd.stream, + version=mmd.version).first() if module: log.debug('Checking whether module build already exist.') - # TODO: make this configurable, we might want to allow - # resubmitting any stuck build on DEV no matter the state + # TODO: make this configurable, we might want to allow + # resubmitting any stuck build on DEV no matter the state if module.state not in (models.BUILD_STATES['failed'],): log.error('Module (state=%s) already exists. ' - 'Only new or failed builds are allowed.' - % module.state) + 'Only new or failed builds are allowed.' + % module.state) raise Conflict('Module (state=%s) already exists. ' - 'Only new or failed builds are allowed.' - % module.state) + 'Only new or failed builds are allowed.' + % module.state) log.debug('Resuming existing module build %r' % module) module.username = username module.transition(conf, models.BUILD_STATES["init"]) log.info("Resumed existing module build in previous state %s" - % module.state) + % module.state) else: log.debug('Creating new module build') module = models.ModuleBuild.create( @@ -296,8 +300,7 @@ def submit_module_build(username, url): existing_build = models.ComponentBuild.query.filter_by( module_id=module.id, package=pkgname).first() - if (existing_build - and existing_build.state != models.BUILD_STATES['done']): + if (existing_build and existing_build.state != models.BUILD_STATES['done']): existing_build.state = models.BUILD_STATES['init'] db.session.add(existing_build) else: From 12973804c8060cab1de2ab16463f94f7e88d1a3a Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:52:09 +0100 Subject: [PATCH 02/24] Network timeouts + lint --- config.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index d5e27caf..283b7d7c 100644 --- a/config.py +++ b/config.py @@ -12,8 +12,12 @@ class BaseConfiguration(object): HOST = '127.0.0.1' PORT = 5000 + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + SYSTEM = 'koji' - MESSAGING = 'fedmsg' # or amq + MESSAGING = 'fedmsg' # or amq KOJI_CONFIG = '/etc/module_build_service/koji.conf' KOJI_PROFILE = 'koji' KOJI_ARCHES = ['i686', 'armv7hl', 'x86_64'] @@ -60,18 +64,22 @@ class BaseConfiguration(object): # AMQ prefixed variables are required only while using 'amq' as messaging backend # Addresses to listen to AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji', - 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service',] + 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service'] # Address for sending messages AMQ_DEST_ADDRESS = 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service' AMQ_CERT_FILE = '/etc/module_build_service/msg-m8y-client.crt' AMQ_PRIVATE_KEY_FILE = '/etc/module_build_service/msg-m8y-client.key' AMQ_TRUSTED_CERT_FILE = '/etc/module_build_service/Root-CA.crt' + class DevConfiguration(BaseConfiguration): LOG_BACKEND = 'console' LOG_LEVEL = 'debug' HOST = '0.0.0.0' + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 if path.exists('/home/fedora/modularity.keytab'): KRB_PRINCIPAL = 'modularity@STG.FEDORAPROJECT.ORG' @@ -86,20 +94,25 @@ class DevConfiguration(BaseConfiguration): REQUIRE_PACKAGER = False # You only need these FAS options if you turn on authorization # with REQUIRE_PACKAGER=True - #FAS_USERNAME = 'put your fas username here' - #FAS_PASSWORD = 'put your fas password here....' - #FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here - #FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip() + # FAS_USERNAME = 'put your fas username here' + # FAS_PASSWORD = 'put your fas password here....' + # FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here + # FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip() KOJI_ARCHES = ['x86_64'] + class TestConfiguration(BaseConfiguration): LOG_BACKEND = 'console' LOG_LEVEL = 'debug' SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:' DEBUG = True + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 + class ProdConfiguration(BaseConfiguration): FAS_USERNAME = 'TODO' - #FAS_PASSWORD = 'another password' + # FAS_PASSWORD = 'another password' From 1cc321a4406e4e430e5bf1756e6ff89953d40033 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:52:23 +0100 Subject: [PATCH 03/24] Default network timeouts --- module_build_service/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module_build_service/config.py b/module_build_service/config.py index f6937dbf..c35943ec 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -200,6 +200,14 @@ class Config(object): 'type': list, 'default': [], 'desc': 'Allowed SCM URLs.'}, + 'net_timeout': { + 'type': int, + 'default': 120, + 'desc': 'Global network timeout for read/write operations, in seconds.'}, + 'net_retry_interval': { + 'type': int, + 'default': 30, + 'desc': 'Global network retry interval for read/write operations, in seconds.'}, } def __init__(self): From 47ccf6d318f4c92aa0473797990992881cdfd203 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:54:57 +0100 Subject: [PATCH 04/24] Uncomment /etc/module_build_service stuff kor koji.conf --- jenkins-check-Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-check-Dockerfile b/jenkins-check-Dockerfile index 756e51a2..debe3491 100644 --- a/jenkins-check-Dockerfile +++ b/jenkins-check-Dockerfile @@ -19,13 +19,13 @@ RUN dnf install -y \ swig \ && dnf autoremove -y \ && dnf clean all \ - && mkdir /opt/module_build_service/ - ###&& mkdir /etc/module_build_service + && mkdir /opt/module_build_service/ \ + && mkdir /etc/module_build_service WORKDIR /opt/module_build_service/ COPY ./requirements.txt /opt/module_build_service/ RUN pip install --user -r ./requirements.txt -###RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf \ +RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf ### && ln -s /opt/module_build_service/copr.conf /etc/module_build_service/copr.conf \ ### && ln -s /opt/module_build_service/krb5-stg.fp.o /etc/krb5.conf.d/stg_fedoraproject_org From f2a8801277dd7892464c2201064a895dc36f133c Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 14:00:50 +0100 Subject: [PATCH 05/24] Lint --- tests/test_builder/test_koji.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 3906433c..82bb1b46 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -28,6 +28,7 @@ import module_build_service.scheduler.handlers.repos import module_build_service.models import module_build_service.builder + class TestKojiBuilder(unittest.TestCase): def setUp(self): @@ -35,7 +36,6 @@ class TestKojiBuilder(unittest.TestCase): self.config.koji_profile = 'staging' self.config.koji_repository_url = 'https://kojipkgs.stg.fedoraproject.org/repos' - def test_tag_to_repo(self): """ Test that when a repo msg hits us and we have no match, that we do nothing gracefully. From 797a339b99eca40c8151466bdda25b5c1fd60bee Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:21:32 +0100 Subject: [PATCH 06/24] add: Testing config values for koji --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index 283b7d7c..ac88a3d2 100644 --- a/config.py +++ b/config.py @@ -112,6 +112,9 @@ class TestConfiguration(BaseConfiguration): NET_TIMEOUT = 5 NET_RETRY_INTERVAL = 1 + KOJI_PROFILE = 'staging' + KOJI_REPOSITORY_URL = 'https://kojipkgs.stg.fedoraproject.org/repos' + class ProdConfiguration(BaseConfiguration): FAS_USERNAME = 'TODO' From 371f3196c9e9c948b692030e4d754b43620404d2 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:21:56 +0100 Subject: [PATCH 07/24] Load defaults from config --- module_build_service/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index b1d66f27..5b3f2174 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -40,7 +40,7 @@ from module_build_service.errors import (Unauthorized, Conflict) from multiprocessing.dummy import Pool as ThreadPool -def retry(timeout=120, interval=30, wait_on=Exception): +def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception): """ A decorator that allows to retry a section of code... ...until success or timeout. """ From d5dfbde5d5ff96aff69b37d0d783821941146fd2 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:22:29 +0100 Subject: [PATCH 08/24] Load config --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index ac8e39b1..7533b39f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -19,11 +19,14 @@ # SOFTWARE. # # Written by Matt Prahl Date: Wed, 16 Nov 2016 15:22:48 +0100 Subject: [PATCH 09/24] PEP8 --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 7533b39f..09401879 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -39,7 +39,7 @@ def init_data(): build_one.stream = '1' build_one.version = 2 build_one.state = 3 - build_one.modulemd = '' # Skipping since no tests rely on it + build_one.modulemd = '' # Skipping since no tests rely on it build_one.koji_tag = 'module-nginx-1.2' build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?' '#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9') From 45cdd608dceaf7e13f960d962b2d33e397cc0929 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:23:40 +0100 Subject: [PATCH 10/24] Load testing values from config --- tests/test_builder/test_koji.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 82bb1b46..1b1df10c 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -28,13 +28,17 @@ import module_build_service.scheduler.handlers.repos import module_build_service.models import module_build_service.builder +from mock import patch + +from tests import conf + class TestKojiBuilder(unittest.TestCase): def setUp(self): self.config = mock.Mock() - self.config.koji_profile = 'staging' - self.config.koji_repository_url = 'https://kojipkgs.stg.fedoraproject.org/repos' + self.config.koji_profile = conf.koji_profile + self.config.koji_repository_url = conf.koji_repository_url def test_tag_to_repo(self): """ Test that when a repo msg hits us and we have no match, From e2e6639ba7622359bb9aaafaca199c73078b2d29 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:24:53 +0100 Subject: [PATCH 11/24] wip: new test for KojiModuleBuilder.buildroot_ready() --- tests/test_builder/test_koji.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 1b1df10c..f79c3b3a 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -50,3 +50,16 @@ class TestKojiBuilder(unittest.TestCase): "x86_64") self.assertEquals(repo, "https://kojipkgs.stg.fedoraproject.org/repos" "/module-base-runtime-0.25-9/latest/x86_64") + + @patch('koji.util.checkForBuilds') + def test_buildroot_ready(self, mocked_kojiutil): + + def mocked_checkForBuilds(*args, **kwargs): + raise IOError + + mocked_kojiutil.return_value.checkForBuilds = mocked_checkForBuilds + kmb = module_build_service.builder.KojiModuleBuilder(owner='Moe Szyslak', + module='nginx', + config=conf, + tag_name='module-nginx-1.2') + kmb.buildroot_ready() From 225d116d459c0d446030be9b3d505485c15b1f0e Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:51:26 +0100 Subject: [PATCH 12/24] lint --- module_build_service/utils.py | 39 +++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index e2e8a342..592a7276 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -20,9 +20,8 @@ # # Written by Ralph Bean # Matt Prahl + """ Utility functions for module_build_service. """ -from flask import request, url_for -from datetime import datetime import re import functools import time @@ -30,13 +29,17 @@ import shutil import tempfile import os import modulemd + +from flask import request, url_for +from datetime import datetime + from module_build_service import log, models from module_build_service.errors import ValidationError, UnprocessableEntity -from module_build_service import app, conf, db, log -from module_build_service.errors import ( - ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound) +from module_build_service import conf, db +from module_build_service.errors import (Unauthorized, Conflict) from multiprocessing.dummy import Pool as ThreadPool + def retry(timeout=120, interval=30, wait_on=Exception): """ A decorator that allows to retry a section of code... ...until success or timeout. @@ -83,7 +86,7 @@ def start_build_batch(config, module, session, builder, components=None): import koji # Placed here to avoid py2/py3 conflicts... if any([c.state == koji.BUILD_STATES['BUILDING'] - for c in module.component_builds ]): + for c in module.component_builds]): raise ValueError("Cannot start a batch when another is in flight.") # The user can either pass in a list of components to 'seed' the batch, or @@ -176,7 +179,7 @@ def filter_module_builds(flask_request): # Filter the query based on date request parameters for item in ('submitted', 'modified', 'completed'): for context in ('before', 'after'): - request_arg = '%s_%s' % (item, context) # i.e. submitted_before + request_arg = '%s_%s' % (item, context) # i.e. submitted_before iso_datetime_arg = request.args.get(request_arg, None) if iso_datetime_arg: @@ -199,6 +202,7 @@ def filter_module_builds(flask_request): per_page = flask_request.args.get('per_page', 10, type=int) return query.paginate(page, per_page, False) + def submit_module_build(username, url): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. @@ -245,24 +249,24 @@ def submit_module_build(username, url): mmd.version = int(scm.version) module = models.ModuleBuild.query.filter_by(name=mmd.name, - stream=mmd.stream, - version=mmd.version).first() + stream=mmd.stream, + version=mmd.version).first() if module: log.debug('Checking whether module build already exist.') - # TODO: make this configurable, we might want to allow - # resubmitting any stuck build on DEV no matter the state + # TODO: make this configurable, we might want to allow + # resubmitting any stuck build on DEV no matter the state if module.state not in (models.BUILD_STATES['failed'],): log.error('Module (state=%s) already exists. ' - 'Only new or failed builds are allowed.' - % module.state) + 'Only new or failed builds are allowed.' + % module.state) raise Conflict('Module (state=%s) already exists. ' - 'Only new or failed builds are allowed.' - % module.state) + 'Only new or failed builds are allowed.' + % module.state) log.debug('Resuming existing module build %r' % module) module.username = username module.transition(conf, models.BUILD_STATES["init"]) log.info("Resumed existing module build in previous state %s" - % module.state) + % module.state) else: log.debug('Creating new module build') module = models.ModuleBuild.create( @@ -339,8 +343,7 @@ def submit_module_build(username, url): existing_build = models.ComponentBuild.query.filter_by( module_id=module.id, package=pkg.name).first() - if (existing_build - and existing_build.state != models.BUILD_STATES['done']): + if (existing_build and existing_build.state != models.BUILD_STATES['done']): existing_build.state = models.BUILD_STATES['init'] db.session.add(existing_build) else: From 708b9a52a8d4fd76f981e6ff2752a7126033d30c Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:52:09 +0100 Subject: [PATCH 13/24] Network timeouts + lint --- config.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index 04eb4a0e..7a13f85f 100644 --- a/config.py +++ b/config.py @@ -12,8 +12,12 @@ class BaseConfiguration(object): HOST = '127.0.0.1' PORT = 5000 + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + SYSTEM = 'koji' - MESSAGING = 'fedmsg' # or amq + MESSAGING = 'fedmsg' # or amq KOJI_CONFIG = '/etc/module_build_service/koji.conf' KOJI_PROFILE = 'koji' KOJI_ARCHES = ['i686', 'armv7hl', 'x86_64'] @@ -64,18 +68,22 @@ class BaseConfiguration(object): # AMQ prefixed variables are required only while using 'amq' as messaging backend # Addresses to listen to AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji', - 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service',] + 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service'] # Address for sending messages AMQ_DEST_ADDRESS = 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service' AMQ_CERT_FILE = '/etc/module_build_service/msg-m8y-client.crt' AMQ_PRIVATE_KEY_FILE = '/etc/module_build_service/msg-m8y-client.key' AMQ_TRUSTED_CERT_FILE = '/etc/module_build_service/Root-CA.crt' + class DevConfiguration(BaseConfiguration): LOG_BACKEND = 'console' LOG_LEVEL = 'debug' HOST = '0.0.0.0' + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 if path.exists('/home/fedora/modularity.keytab'): KRB_PRINCIPAL = 'modularity@STG.FEDORAPROJECT.ORG' @@ -90,20 +98,25 @@ class DevConfiguration(BaseConfiguration): REQUIRE_PACKAGER = False # You only need these FAS options if you turn on authorization # with REQUIRE_PACKAGER=True - #FAS_USERNAME = 'put your fas username here' - #FAS_PASSWORD = 'put your fas password here....' - #FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here - #FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip() + # FAS_USERNAME = 'put your fas username here' + # FAS_PASSWORD = 'put your fas password here....' + # FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here + # FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip() KOJI_ARCHES = ['x86_64'] + class TestConfiguration(BaseConfiguration): LOG_BACKEND = 'console' LOG_LEVEL = 'debug' SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:' DEBUG = True + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 + class ProdConfiguration(BaseConfiguration): FAS_USERNAME = 'TODO' - #FAS_PASSWORD = 'another password' + # FAS_PASSWORD = 'another password' From ca198daebf9b04f158e8adea57fc5c24ef26f2f2 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:52:23 +0100 Subject: [PATCH 14/24] Default network timeouts --- module_build_service/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/module_build_service/config.py b/module_build_service/config.py index d199f3be..af8d0a1e 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -29,7 +29,6 @@ from module_build_service import app from module_build_service import logger - def from_app_config(): """ Create the configuration instance from the values in app.config """ @@ -208,6 +207,14 @@ class Config(object): 'type': int, 'default': 0, 'desc': 'Number of consecutive component builds.'}, + 'net_timeout': { + 'type': int, + 'default': 120, + 'desc': 'Global network timeout for read/write operations, in seconds.'}, + 'net_retry_interval': { + 'type': int, + 'default': 30, + 'desc': 'Global network retry interval for read/write operations, in seconds.'}, } def __init__(self): From 2227d883331656b25939a988679d29fbe093c808 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 13:54:57 +0100 Subject: [PATCH 15/24] Uncomment /etc/module_build_service stuff kor koji.conf --- jenkins-check-Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins-check-Dockerfile b/jenkins-check-Dockerfile index 756e51a2..debe3491 100644 --- a/jenkins-check-Dockerfile +++ b/jenkins-check-Dockerfile @@ -19,13 +19,13 @@ RUN dnf install -y \ swig \ && dnf autoremove -y \ && dnf clean all \ - && mkdir /opt/module_build_service/ - ###&& mkdir /etc/module_build_service + && mkdir /opt/module_build_service/ \ + && mkdir /etc/module_build_service WORKDIR /opt/module_build_service/ COPY ./requirements.txt /opt/module_build_service/ RUN pip install --user -r ./requirements.txt -###RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf \ +RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf ### && ln -s /opt/module_build_service/copr.conf /etc/module_build_service/copr.conf \ ### && ln -s /opt/module_build_service/krb5-stg.fp.o /etc/krb5.conf.d/stg_fedoraproject_org From 2a53b74bcf237763a4166eefb7f7f5de644d0333 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 14:00:50 +0100 Subject: [PATCH 16/24] Lint --- tests/test_builder/test_koji.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 3906433c..82bb1b46 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -28,6 +28,7 @@ import module_build_service.scheduler.handlers.repos import module_build_service.models import module_build_service.builder + class TestKojiBuilder(unittest.TestCase): def setUp(self): @@ -35,7 +36,6 @@ class TestKojiBuilder(unittest.TestCase): self.config.koji_profile = 'staging' self.config.koji_repository_url = 'https://kojipkgs.stg.fedoraproject.org/repos' - def test_tag_to_repo(self): """ Test that when a repo msg hits us and we have no match, that we do nothing gracefully. From 951a54576fdd3c070add56e4b0d5497591402ec0 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:21:32 +0100 Subject: [PATCH 17/24] add: Testing config values for koji --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index 7a13f85f..592be47c 100644 --- a/config.py +++ b/config.py @@ -116,6 +116,9 @@ class TestConfiguration(BaseConfiguration): NET_TIMEOUT = 5 NET_RETRY_INTERVAL = 1 + KOJI_PROFILE = 'staging' + KOJI_REPOSITORY_URL = 'https://kojipkgs.stg.fedoraproject.org/repos' + class ProdConfiguration(BaseConfiguration): FAS_USERNAME = 'TODO' From d8942ba44a0a0de438d6db16c4fa7eb5abc50446 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:21:56 +0100 Subject: [PATCH 18/24] Load defaults from config --- module_build_service/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 592a7276..48ac5dbd 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -40,7 +40,7 @@ from module_build_service.errors import (Unauthorized, Conflict) from multiprocessing.dummy import Pool as ThreadPool -def retry(timeout=120, interval=30, wait_on=Exception): +def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception): """ A decorator that allows to retry a section of code... ...until success or timeout. """ From 62b4a34ffd171f0e56a271c8e3cdda76d895d5b6 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:22:29 +0100 Subject: [PATCH 19/24] Load config --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index ac8e39b1..7533b39f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -19,11 +19,14 @@ # SOFTWARE. # # Written by Matt Prahl Date: Wed, 16 Nov 2016 15:22:48 +0100 Subject: [PATCH 20/24] PEP8 --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 7533b39f..09401879 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -39,7 +39,7 @@ def init_data(): build_one.stream = '1' build_one.version = 2 build_one.state = 3 - build_one.modulemd = '' # Skipping since no tests rely on it + build_one.modulemd = '' # Skipping since no tests rely on it build_one.koji_tag = 'module-nginx-1.2' build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?' '#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9') From 37cde8cf2fb846c389da0e2784cfc037297274ad Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:23:40 +0100 Subject: [PATCH 21/24] Load testing values from config --- tests/test_builder/test_koji.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 82bb1b46..1b1df10c 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -28,13 +28,17 @@ import module_build_service.scheduler.handlers.repos import module_build_service.models import module_build_service.builder +from mock import patch + +from tests import conf + class TestKojiBuilder(unittest.TestCase): def setUp(self): self.config = mock.Mock() - self.config.koji_profile = 'staging' - self.config.koji_repository_url = 'https://kojipkgs.stg.fedoraproject.org/repos' + self.config.koji_profile = conf.koji_profile + self.config.koji_repository_url = conf.koji_repository_url def test_tag_to_repo(self): """ Test that when a repo msg hits us and we have no match, From 35781690a7d47093663969c2fbab103bf4f0a1b2 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 16 Nov 2016 15:24:53 +0100 Subject: [PATCH 22/24] wip: new test for KojiModuleBuilder.buildroot_ready() --- tests/test_builder/test_koji.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 1b1df10c..f79c3b3a 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -50,3 +50,16 @@ class TestKojiBuilder(unittest.TestCase): "x86_64") self.assertEquals(repo, "https://kojipkgs.stg.fedoraproject.org/repos" "/module-base-runtime-0.25-9/latest/x86_64") + + @patch('koji.util.checkForBuilds') + def test_buildroot_ready(self, mocked_kojiutil): + + def mocked_checkForBuilds(*args, **kwargs): + raise IOError + + mocked_kojiutil.return_value.checkForBuilds = mocked_checkForBuilds + kmb = module_build_service.builder.KojiModuleBuilder(owner='Moe Szyslak', + module='nginx', + config=conf, + tag_name='module-nginx-1.2') + kmb.buildroot_ready() From 99c3c1e71780b25bb5ec2840c1771395f2eb414f Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Fri, 18 Nov 2016 14:37:54 +0100 Subject: [PATCH 23/24] catch on IOError and retry --- module_build_service/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index 7eae1399..d66eaf48 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -321,7 +321,7 @@ class KojiModuleBuilder(GenericBuilder): return "" % ( self.module_str, self.tag_name) - @module_build_service.utils.retry(wait_on=koji.GenericError) + @module_build_service.utils.retry(wait_on=(IOError, koji.GenericError)) def buildroot_ready(self, artifacts=None): """ :param artifacts=None - list of nvrs From 9b808d43549d0ef65adbdeb40bdfcc22b7426a6b Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Fri, 18 Nov 2016 14:39:27 +0100 Subject: [PATCH 24/24] test buildroot_read() incl. FakeKojiModuleBuilder and FakeKojiSession --- tests/test_builder/test_koji.py | 54 +++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index f79c3b3a..43c7a0ca 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -21,7 +21,10 @@ # Written by Jan Kaluza import unittest +import munch import mock +import koji +import xmlrpclib import module_build_service.messaging import module_build_service.scheduler.handlers.repos @@ -32,6 +35,8 @@ from mock import patch from tests import conf +from module_build_service.builder import KojiModuleBuilder + class TestKojiBuilder(unittest.TestCase): @@ -51,15 +56,46 @@ class TestKojiBuilder(unittest.TestCase): self.assertEquals(repo, "https://kojipkgs.stg.fedoraproject.org/repos" "/module-base-runtime-0.25-9/latest/x86_64") - @patch('koji.util.checkForBuilds') + @patch('koji.util') def test_buildroot_ready(self, mocked_kojiutil): - def mocked_checkForBuilds(*args, **kwargs): - raise IOError + attrs = {'checkForBuilds.return_value': None, + 'checkForBuilds.side_effect': IOError} + mocked_kojiutil.configure_mock(**attrs) + fake_kmb = FakeKojiModuleBuilder(owner='Moe Szyslak', + module='nginx', + config=conf, + tag_name='module-nginx-1.2') + fake_kmb.module_target = {'build_tag': 'fake_tag'} - mocked_kojiutil.return_value.checkForBuilds = mocked_checkForBuilds - kmb = module_build_service.builder.KojiModuleBuilder(owner='Moe Szyslak', - module='nginx', - config=conf, - tag_name='module-nginx-1.2') - kmb.buildroot_ready() + with self.assertRaises(IOError): + fake_kmb.buildroot_ready() + self.assertEquals(mocked_kojiutil.checkForBuilds.call_count, 5) + + +class FakeKojiModuleBuilder(KojiModuleBuilder): + + @module_build_service.utils.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError)) + def get_session(self, config, owner): + koji_config = munch.Munch(koji.read_config( + profile_name=config.koji_profile, + user_config=config.koji_config, + )) + + address = koji_config.server + + koji_session = FakeKojiSession(address, opts=koji_config) + + return koji_session + + +class FakeKojiSession(koji.ClientSession): + + def _callMethod(self, name, args, kwargs=None): + pass + + def _setup_connection(self): + pass + + def getRepo(self, tag): + return {'create_event': 'fake event'}