modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 11:48:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Try to build testmodule.yaml as unit-test with fake builder backend.

Browse Source
This commit is contained in:
Jan Kaluza
2016-11-22 14:05:49 +01:00
parent c719e1d0c1
commit 77366b94ce
4 changed files with 275 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
module_build_service/builder.py
View File

@@ -107,6 +107,11 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
"""
backend = "generic"
backends = {}
@classmethod
def register_backend_class(cls, backend_class):
GenericBuilder.backends[backend_class.backend] = backend_class
@classmethod
def create(cls, owner, module, backend, config, **extra):
@@ -123,14 +128,8 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
if isinstance(config.system, Mock):
return KojiModuleBuilder(owner=owner, module=module,
config=config, **extra)
elif backend == "koji":
return KojiModuleBuilder(owner=owner, module=module,
config=config, **extra)
elif backend == "copr":
return CoprModuleBuilder(owner=owner, module=module,
config=config, **extra)
elif backend == "mock":
return MockModuleBuilder(owner=owner, module=module,
elif backend in GenericBuilder.backends:
return GenericBuilder.backends[backend](owner=owner, module=module,
config=config, **extra)
else:
raise ValueError("Builder backend='%s' not recognized" % backend)
@@ -146,10 +145,9 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
Returns URL of repository containing the built artifacts for
the tag with particular name and architecture.
"""
if backend == "koji":
return KojiModuleBuilder.repo_from_tag(config, tag_name, arch)
if backend == "copr":
return CoprModuleBuilder.repo_from_tag(config, tag_name, arch)
if backend in GenericBuilder.backends:
return GenericBuilder.backends[backend].repo_from_tag(
config, tag_name, arch)
else:
raise ValueError("Builder backend='%s' not recognized" % backend)
@@ -1100,3 +1098,7 @@ class MockModuleBuilder(GenericBuilder):
def get_disttag_srpm(disttag):
# @FIXME
return KojiModuleBuilder.get_disttag_srpm(disttag)
GenericBuilder.register_backend_class(KojiModuleBuilder)
GenericBuilder.register_backend_class(CoprModuleBuilder)
GenericBuilder.register_backend_class(MockModuleBuilder)

25
module_build_service/scheduler/main.py
View File

@@ -65,15 +65,21 @@ def module_build_state_from_msg(msg):
class MessageIngest(threading.Thread):
def __init__(self, outgoing_work_queue, *args, **kwargs):
def __init__(self, outgoing_work_queue, stop_after_build, *args, **kwargs):
self.outgoing_work_queue = outgoing_work_queue
super(MessageIngest, self).__init__(*args, **kwargs)
self.stop_after_build = stop_after_build
def run(self):
for msg in module_build_service.messaging.listen(conf):
self.outgoing_work_queue.put(msg)
if type(msg) == module_build_service.messaging.RidaModule:
if (self.stop_after_build and module_build_state_from_msg(msg)
in [models.BUILD_STATES["failed"], models.BUILD_STATES["ready"]]):
break
class MessageWorker(threading.Thread):
@@ -130,7 +136,6 @@ class MessageWorker(threading.Thread):
if msg is STOP_WORK:
log.info("Worker thread received STOP_WORK, shutting down...")
os._exit(0)
break
try:
@@ -183,9 +188,10 @@ class Poller(threading.Thread):
def __init__(self, outgoing_work_queue, *args, **kwargs):
self.outgoing_work_queue = outgoing_work_queue
super(Poller, self).__init__(*args, **kwargs)
self.stop = False
def run(self):
while True:
while not self.stop:
with models.make_session(conf) as session:
self.log_summary(session)
# XXX: detect whether it's really stucked first
@@ -195,7 +201,10 @@ class Poller(threading.Thread):
self.process_paused_module_builds(conf, session)
log.info("Polling thread sleeping, %rs" % conf.polling_interval)
time.sleep(conf.polling_interval)
for i in range(0, conf.polling_interval):
time.sleep(1)
if self.stop:
break
def fail_lost_builds(self, session):
# This function is supposed to be handling only
@@ -245,9 +254,6 @@ class Poller(threading.Thread):
elif conf.system == "mock":
pass
else:
raise NotImplementedError("Buildsystem %r is not supported." % conf.system)
def log_summary(self, session):
log.info("Current status:")
backlog = self.outgoing_work_queue.qsize()
@@ -312,7 +318,7 @@ def main(initial_msgs = [], return_after_build = False):
try:
# This ingest thread puts work on the queue
messaging_thread = MessageIngest(_work_queue)
messaging_thread = MessageIngest(_work_queue, return_after_build)
# This poller does other work, but also sometimes puts work in queue.
polling_thread = Poller(_work_queue)
# This worker takes work off the queue and handles it.
@@ -322,6 +328,9 @@ def main(initial_msgs = [], return_after_build = False):
polling_thread.start()
worker_thread.start()
worker_thread.join()
polling_thread.stop = True
except KeyboardInterrupt:
# FIXME: Make this less brutal
os._exit(0)

205
tests/test_build/test_build.py Normal file
View File

@@ -0,0 +1,205 @@
# Copyright (c) 2016 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Jan Kaluza <jkaluza@redhat.com>
import unittest
import munch
import mock
import koji
import xmlrpclib
from os import path, mkdir
from shutil import copyfile
from module_build_service import db
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
from module_build_service import models, conf
from module_build_service.utils import submit_module_build
from module_build_service.messaging import RidaModule
from mock import patch
from tests import app, init_data
from tests import conf as test_conf
import json
from module_build_service.builder import KojiModuleBuilder, GenericBuilder
import module_build_service.scheduler.main
class MockedSCM(object):
def __init__(self, mocked_scm, name, mmd_filename):
self.mocked_scm = mocked_scm
self.name = name
self.mmd_filename = mmd_filename
self.mocked_scm.return_value.checkout = self.checkout
self.mocked_scm.return_value.name = self.name
self.mocked_scm.return_value.get_latest = self.get_latest
def checkout(self, temp_dir):
scm_dir = path.join(temp_dir, self.name)
mkdir(scm_dir)
base_dir = path.abspath(path.dirname(__file__))
copyfile(path.join(base_dir, self.mmd_filename),
path.join(scm_dir, self.mmd_filename))
return scm_dir
def get_latest(self, branch = 'master'):
return branch
class TestModuleBuilder(GenericBuilder):
"""
Test module builder which succeeds for every build.
"""
backend = "mock"
# Global build_id/task_id we increment when new build is executed.
_build_id = 1
def __init__(self, owner, module, config, tag_name):
self.module_str = module
self.tag_name = tag_name
self.config = config
def buildroot_connect(self, groups):
pass
def buildroot_prep(self):
pass
def buildroot_resume(self):
pass
def buildroot_ready(self, artifacts=None):
return True
def buildroot_add_dependency(self, dependencies):
pass
def buildroot_add_artifacts(self, artifacts, install=False):
pass
def buildroot_add_repos(self, dependencies):
pass
def _send_repo_done(self):
msg = module_build_service.messaging.KojiRepoChange(
msg_id='a faked internal message',
repo_tag=self.tag_name + "-build",
)
module_build_service.scheduler.main.outgoing_work_queue_put(msg)
def _send_build_change(self, state, source, build_id):
# build_id=1 and task_id=1 are OK here, because we are building just
# one RPM at the time.
msg = module_build_service.messaging.KojiBuildChange(
msg_id='a faked internal message',
build_id=build_id,
task_id=build_id,
build_name="name",
build_new_state=state,
build_release="1",
build_version="1"
)
module_build_service.scheduler.main.outgoing_work_queue_put(msg)
def build(self, artifact_name, source):
print "Starting building artifact %s: %s" % (artifact_name, source)
TestModuleBuilder._build_id += 1
self._send_repo_done()
self._send_build_change(koji.BUILD_STATES['COMPLETE'], source,
TestModuleBuilder._build_id)
self._send_repo_done()
state = koji.BUILD_STATES['BUILDING']
reason = "Submitted %s to Koji" % (artifact_name)
return TestModuleBuilder._build_id, state, reason, None
@staticmethod
def get_disttag_srpm(disttag):
# @FIXME
return KojiModuleBuilder.get_disttag_srpm(disttag)
def set_dburi(dburi):
"""
Sets database URI in all places in the middle of test.
"""
conf.set_item("sqlalchemy_database_uri", dburi)
test_conf.set_item("sqlalchemy_database_uri", dburi)
app.config["SQLALCHEMY_DATABASE_URI"] = dburi
class TestBuild(unittest.TestCase):
def setUp(self):
GenericBuilder.register_backend_class(TestModuleBuilder)
self.client = app.test_client()
conf.set_item("system", "mock")
# We need to use real database on fileystem for these tests, because
# there might be multiple threads and processes accessing it
# and in-memory database wouldn't work.
self.orig_dburi = app.config["SQLALCHEMY_DATABASE_URI"]
dbdir = path.abspath(path.dirname(__file__))
dburi = 'sqlite:///{0}'.format(path.join(
dbdir, '.test_module_build_service.db'))
set_dburi(dburi)
init_data()
models.ModuleBuild.query.delete()
models.ComponentBuild.query.delete()
def tearDown(self):
# Set back the original database URI
set_dburi(self.orig_dburi)
conf.set_item("system", "koji")
@patch('module_build_service.auth.get_username', return_value='Homer J. Simpson')
@patch('module_build_service.auth.assert_is_packager')
@patch('module_build_service.scm.SCM')
def test_submit_build(self, mocked_scm, mocked_assert_is_packager,
mocked_get_username):
"""
Tests the build of testmodule.yaml using TestModuleBuilder which
succeeds everytime.
"""
mocked_scm_obj = MockedSCM(mocked_scm, "testmodule", "testmodule.yaml")
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
module_build_id = data['id']
msgs = []
msgs.append(RidaModule("fake msg", 1, 1))
module_build_service.scheduler.main.main(msgs, True)
# All components should be built and module itself should be in "done"
# or "ready" state.
for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE'])
self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )

39
tests/test_build/testmodule.yaml Normal file
View File

@@ -0,0 +1,39 @@
document: modulemd
version: 1
data:
summary: A test module in all its beauty
description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline.
name: testmodule
stream: teststream
version: 1
license:
module: [ MIT ]
dependencies:
buildrequires:
base-runtime: master
requires:
base-runtime: master
references:
community: https://fedoraproject.org/wiki/Modularity
documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
tracker: https://taiga.fedorainfracloud.org/project/modularity
profiles:
default:
rpms:
- tangerine
api:
rpms:
- perl-Tangerine
- tangerine
components:
rpms:
perl-List-Compare:
rationale: A dependency of tangerine.
ref: f25
perl-Tangerine:
rationale: Provides API for this module and is a dependency of tangerine.
ref: f25
tangerine:
rationale: Provides API for this module.
buildorder: 10
ref: f25