modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-06-15 06:27:00 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #196 Fix #94 - handle IOError

Browse Source
This commit is contained in:
Filip Valder
2016-11-18 14:11:20 +00:00
parent b2b63e6a1f 9b808d4354
commit 8c6a5f4919
7 changed files with 117 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
config.py
View File

@@ -12,8 +12,12 @@ class BaseConfiguration(object):
HOST = '127.0.0.1'
PORT = 5000
# Global network-related values, in seconds
NET_TIMEOUT = 120
NET_RETRY_INTERVAL = 30
SYSTEM = 'koji'
MESSAGING = 'fedmsg' # or amq
MESSAGING = 'fedmsg' # or amq
KOJI_CONFIG = '/etc/module_build_service/koji.conf'
KOJI_PROFILE = 'koji'
KOJI_ARCHES = ['i686', 'armv7hl', 'x86_64']
@@ -64,18 +68,22 @@ class BaseConfiguration(object):
# AMQ prefixed variables are required only while using 'amq' as messaging backend
# Addresses to listen to
AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji',
'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service',]
'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service']
# Address for sending messages
AMQ_DEST_ADDRESS = 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service'
AMQ_CERT_FILE = '/etc/module_build_service/msg-m8y-client.crt'
AMQ_PRIVATE_KEY_FILE = '/etc/module_build_service/msg-m8y-client.key'
AMQ_TRUSTED_CERT_FILE = '/etc/module_build_service/Root-CA.crt'
class DevConfiguration(BaseConfiguration):
LOG_BACKEND = 'console'
LOG_LEVEL = 'debug'
HOST = '0.0.0.0'
# Global network-related values, in seconds
NET_TIMEOUT = 5
NET_RETRY_INTERVAL = 1
if path.exists('/home/fedora/modularity.keytab'):
KRB_PRINCIPAL = 'modularity@STG.FEDORAPROJECT.ORG'
@@ -90,20 +98,28 @@ class DevConfiguration(BaseConfiguration):
REQUIRE_PACKAGER = False
# You only need these FAS options if you turn on authorization
# with REQUIRE_PACKAGER=True
#FAS_USERNAME = 'put your fas username here'
#FAS_PASSWORD = 'put your fas password here....'
#FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here
#FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip()
# FAS_USERNAME = 'put your fas username here'
# FAS_PASSWORD = 'put your fas password here....'
# FAS_PASSWORD = os.environ('FAS_PASSWORD') # you could store it here
# FAS_PASSWORD = commands.getoutput('pass your_fas_password').strip()
KOJI_ARCHES = ['x86_64']
class TestConfiguration(BaseConfiguration):
LOG_BACKEND = 'console'
LOG_LEVEL = 'debug'
SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
DEBUG = True
# Global network-related values, in seconds
NET_TIMEOUT = 5
NET_RETRY_INTERVAL = 1
KOJI_PROFILE = 'staging'
KOJI_REPOSITORY_URL = 'https://kojipkgs.stg.fedoraproject.org/repos'
class ProdConfiguration(BaseConfiguration):
FAS_USERNAME = 'TODO'
#FAS_PASSWORD = 'another password'
# FAS_PASSWORD = 'another password'

6
jenkins-check-Dockerfile
View File

@@ -19,13 +19,13 @@ RUN dnf install -y \
swig \
&& dnf autoremove -y \
&& dnf clean all \
&& mkdir /opt/module_build_service/
###&& mkdir /etc/module_build_service
&& mkdir /opt/module_build_service/ \
&& mkdir /etc/module_build_service
WORKDIR /opt/module_build_service/
COPY ./requirements.txt /opt/module_build_service/
RUN pip install --user -r ./requirements.txt
###RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf \
RUN ln -s /opt/module_build_service/koji.conf /etc/module_build_service/koji.conf
### && ln -s /opt/module_build_service/copr.conf /etc/module_build_service/copr.conf \
### && ln -s /opt/module_build_service/krb5-stg.fp.o /etc/krb5.conf.d/stg_fedoraproject_org

2
module_build_service/builder.py
View File

@@ -321,7 +321,7 @@ class KojiModuleBuilder(GenericBuilder):
return "<KojiModuleBuilder module: %s, tag: %s>" % (
self.module_str, self.tag_name)
@module_build_service.utils.retry(wait_on=koji.GenericError)
@module_build_service.utils.retry(wait_on=(IOError, koji.GenericError))
def buildroot_ready(self, artifacts=None):
"""
:param artifacts=None - list of nvrs

9
module_build_service/config.py
View File

@@ -29,7 +29,6 @@ from module_build_service import app
from module_build_service import logger
def from_app_config():
""" Create the configuration instance from the values in app.config
"""
@@ -208,6 +207,14 @@ class Config(object):
'type': int,
'default': 0,
'desc': 'Number of consecutive component builds.'},
'net_timeout': {
'type': int,
'default': 120,
'desc': 'Global network timeout for read/write operations, in seconds.'},
'net_retry_interval': {
'type': int,
'default': 30,
'desc': 'Global network retry interval for read/write operations, in seconds.'},
}
def __init__(self):

41
module_build_service/utils.py
View File

@@ -20,9 +20,8 @@
#
# Written by Ralph Bean <rbean@redhat.com>
# Matt Prahl <mprahl@redhat.com>
""" Utility functions for module_build_service. """
from flask import request, url_for
from datetime import datetime
import re
import functools
import time
@@ -30,14 +29,18 @@ import shutil
import tempfile
import os
import modulemd
from flask import request, url_for
from datetime import datetime
from module_build_service import log, models
from module_build_service.errors import ValidationError, UnprocessableEntity
from module_build_service import app, conf, db, log
from module_build_service.errors import (
ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound)
from module_build_service import conf, db
from module_build_service.errors import (Unauthorized, Conflict)
from multiprocessing.dummy import Pool as ThreadPool
def retry(timeout=120, interval=30, wait_on=Exception):
def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception):
""" A decorator that allows to retry a section of code...
...until success or timeout.
"""
@@ -83,7 +86,7 @@ def start_build_batch(config, module, session, builder, components=None):
import koji # Placed here to avoid py2/py3 conflicts...
if any([c.state == koji.BUILD_STATES['BUILDING']
for c in module.component_builds ]):
for c in module.component_builds]):
raise ValueError("Cannot start a batch when another is in flight.")
# The user can either pass in a list of components to 'seed' the batch, or
@@ -176,7 +179,7 @@ def filter_module_builds(flask_request):
# Filter the query based on date request parameters
for item in ('submitted', 'modified', 'completed'):
for context in ('before', 'after'):
request_arg = '%s_%s' % (item, context) # i.e. submitted_before
request_arg = '%s_%s' % (item, context) # i.e. submitted_before
iso_datetime_arg = request.args.get(request_arg, None)
if iso_datetime_arg:
@@ -199,6 +202,7 @@ def filter_module_builds(flask_request):
per_page = flask_request.args.get('per_page', 10, type=int)
return query.paginate(page, per_page, False)
def submit_module_build(username, url):
# Import it here, because SCM uses utils methods
# and fails to import them because of dep-chain.
@@ -245,24 +249,24 @@ def submit_module_build(username, url):
mmd.version = int(scm.version)
module = models.ModuleBuild.query.filter_by(name=mmd.name,
stream=mmd.stream,
version=mmd.version).first()
stream=mmd.stream,
version=mmd.version).first()
if module:
log.debug('Checking whether module build already exist.')
# TODO: make this configurable, we might want to allow
# resubmitting any stuck build on DEV no matter the state
# TODO: make this configurable, we might want to allow
# resubmitting any stuck build on DEV no matter the state
if module.state not in (models.BUILD_STATES['failed'],):
log.error('Module (state=%s) already exists. '
'Only new or failed builds are allowed.'
% module.state)
'Only new or failed builds are allowed.'
% module.state)
raise Conflict('Module (state=%s) already exists. '
'Only new or failed builds are allowed.'
% module.state)
'Only new or failed builds are allowed.'
% module.state)
log.debug('Resuming existing module build %r' % module)
module.username = username
module.transition(conf, models.BUILD_STATES["init"])
log.info("Resumed existing module build in previous state %s"
% module.state)
% module.state)
else:
log.debug('Creating new module build')
module = models.ModuleBuild.create(
@@ -345,8 +349,7 @@ def submit_module_build(username, url):
existing_build = models.ComponentBuild.query.filter_by(
module_id=module.id, package=pkg.name).first()
if (existing_build
and existing_build.state != models.BUILD_STATES['done']):
if (existing_build and existing_build.state != models.BUILD_STATES['done']):
existing_build.state = models.BUILD_STATES['init']
db.session.add(existing_build)
else:

5
tests/__init__.py
View File

@@ -19,11 +19,14 @@
# SOFTWARE.
#
# Written by Matt Prahl <mprahl@redhat.com
from datetime import datetime, timedelta
from module_build_service import app, db
from module_build_service.config import from_app_config
from module_build_service.models import ModuleBuild, ComponentBuild
app.config.from_object('config.TestConfiguration')
conf = from_app_config()
def init_data():
@@ -36,7 +39,7 @@ def init_data():
build_one.stream = '1'
build_one.version = 2
build_one.state = 3
build_one.modulemd = '' # Skipping since no tests rely on it
build_one.modulemd = '' # Skipping since no tests rely on it
build_one.koji_tag = 'module-nginx-1.2'
build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?'
'#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9')

59
tests/test_builder/test_koji.py
View File

@@ -21,20 +21,29 @@
# Written by Jan Kaluza <jkaluza@redhat.com>
import unittest
import munch
import mock
import koji
import xmlrpclib
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
import module_build_service.models
import module_build_service.builder
from mock import patch
from tests import conf
from module_build_service.builder import KojiModuleBuilder
class TestKojiBuilder(unittest.TestCase):
def setUp(self):
self.config = mock.Mock()
self.config.koji_profile = 'staging'
self.config.koji_repository_url = 'https://kojipkgs.stg.fedoraproject.org/repos'
self.config.koji_profile = conf.koji_profile
self.config.koji_repository_url = conf.koji_repository_url
def test_tag_to_repo(self):
""" Test that when a repo msg hits us and we have no match,
@@ -46,3 +55,47 @@ class TestKojiBuilder(unittest.TestCase):
"x86_64")
self.assertEquals(repo, "https://kojipkgs.stg.fedoraproject.org/repos"
"/module-base-runtime-0.25-9/latest/x86_64")
@patch('koji.util')
def test_buildroot_ready(self, mocked_kojiutil):
attrs = {'checkForBuilds.return_value': None,
'checkForBuilds.side_effect': IOError}
mocked_kojiutil.configure_mock(**attrs)
fake_kmb = FakeKojiModuleBuilder(owner='Moe Szyslak',
module='nginx',
config=conf,
tag_name='module-nginx-1.2')
fake_kmb.module_target = {'build_tag': 'fake_tag'}
with self.assertRaises(IOError):
fake_kmb.buildroot_ready()
self.assertEquals(mocked_kojiutil.checkForBuilds.call_count, 5)
class FakeKojiModuleBuilder(KojiModuleBuilder):
@module_build_service.utils.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError))
def get_session(self, config, owner):
koji_config = munch.Munch(koji.read_config(
profile_name=config.koji_profile,
user_config=config.koji_config,
))
address = koji_config.server
koji_session = FakeKojiSession(address, opts=koji_config)
return koji_session
class FakeKojiSession(koji.ClientSession):
def _callMethod(self, name, args, kwargs=None):
pass
def _setup_connection(self):
pass
def getRepo(self, tag):
return {'create_event': 'fake event'}