modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-04 19:28:49 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #123 Updating code to allow for componentless module builds

Browse Source
This commit is contained in:
Courtney Pacheco
2016-10-25 14:03:49 +00:00
parent 4a615d5144 8e4af4c985
commit 32c623769e
6 changed files with 113 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
module_build_service/pdc.py
View File

@@ -29,7 +29,7 @@ import modulemd
from pdc_client import PDCClient
from copr.client import CoprClient
import six
import rida
import module_build_service
@@ -155,7 +155,7 @@ def get_module_tag(session, module_info, strict=False):
"""
return get_module(session, module_info, strict=strict)['koji_tag']
def get_module_repo(session, module_info, strict=False, config=rida.conf):
def get_module_repo(session, module_info, strict=False, config=module_build_service.conf):
"""
:param session : PDCClient instance
:param module_info: list of module_info dicts

7
module_build_service/scheduler/handlers/components.py
View File

@@ -90,6 +90,13 @@ def _finalize(config, session, msg, state):
# Find all of the sibling builds of this particular build.
current_batch = parent.current_batch()
# If there are no components, transition module build state to 'done'
if not current_batch:
log.info("Batch is empty. Moving module state to DONE.")
parent.transition(config, module.BUILD_STATES['done'])
session.commit()
return
# Otherwise, check to see if all failed. If so, then we cannot continue on
# to a next batch. This module build is doomed.
if all([c.state != koji.BUILD_STATES['COMPLETE'] for c in current_batch]):

110
module_build_service/views.py
View File

@@ -166,66 +166,64 @@ class ModuleBuildAPI(MethodView):
# the availability of git URLs paralelly later.
full_urls = []
# List of (pkg_name, git_url) tuples to be used to check
# the availability of git URLs paralelly later.
full_urls = []
# If the modulemd yaml specifies components, then submit them for build
if mmd.components:
for pkgname, pkg in mmd.components.rpms.packages.items():
try:
if pkg.get("repository") and not conf.rpms_allow_repository:
raise Unauthorized(
"Custom component repositories aren't allowed")
if pkg.get("cache") and not conf.rpms_allow_cache:
raise Unauthorized("Custom component caches aren't allowed")
if not pkg.get("repository"):
pkg["repository"] = conf.rpms_default_repository + pkgname
if not pkg.get("cache"):
pkg["cache"] = conf.rpms_default_cache + pkgname
if not pkg.get("commit"):
try:
pkg["commit"] = module_build_service.scm.SCM(
pkg["repository"]).get_latest()
except Exception as e:
raise UnprocessableEntity(
"Failed to get the latest commit: %s" % pkgname)
except Exception:
module.transition(conf, models.BUILD_STATES["failed"])
db.session.add(module)
db.session.commit()
raise
for pkgname, pkg in mmd.components.rpms.packages.items():
try:
if pkg.get("repository") and not conf.rpms_allow_repository:
raise Unauthorized(
"Custom component repositories aren't allowed")
if pkg.get("cache") and not conf.rpms_allow_cache:
raise Unauthorized("Custom component caches aren't allowed")
if not pkg.get("repository"):
pkg["repository"] = conf.rpms_default_repository + pkgname
if not pkg.get("cache"):
pkg["cache"] = conf.rpms_default_cache + pkgname
if not pkg.get("commit"):
try:
pkg["commit"] = module_build_service.scm.SCM(
pkg["repository"]).get_latest()
except Exception as e:
raise UnprocessableEntity(
"Failed to get the latest commit: %s" % pkgname)
except Exception:
module.transition(conf, models.BUILD_STATES["failed"])
db.session.add(module)
db.session.commit()
raise
full_url = pkg["repository"] + "?#" + pkg["commit"]
full_urls.append((pkgname, full_url))
full_url = pkg["repository"] + "?#" + pkg["commit"]
full_urls.append((pkgname, full_url))
log.debug("Checking scm urls")
# Checks the availability of SCM urls.
pool = ThreadPool(10)
err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0])
if not module_build_service.scm.SCM(data[1]).is_available()
else None, full_urls)
for err_msg in err_msgs:
if err_msg:
raise UnprocessableEntity(err_msg)
log.debug("Checking scm urls")
# Checks the availability of SCM urls.
pool = ThreadPool(10)
err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0])
if not module_build_service.scm.SCM(data[1]).is_available()
else None, full_urls)
for err_msg in err_msgs:
if err_msg:
raise UnprocessableEntity(err_msg)
for pkgname, pkg in mmd.components.rpms.packages.items():
full_url = pkg["repository"] + "?#" + pkg["commit"]
for pkgname, pkg in mmd.components.rpms.packages.items():
full_url = pkg["repository"] + "?#" + pkg["commit"]
existing_build = models.ComponentBuild.query.filter_by(
module_id=module.id, package=pkgname).first()
if (existing_build
and existing_build.state != models.BUILD_STATES['done']):
existing_build.state = models.BUILD_STATES['init']
db.session.add(existing_build)
else:
# XXX: what about components that were present in previous
# builds but are gone now (component reduction)?
build = models.ComponentBuild(
module_id=module.id,
package=pkgname,
format="rpms",
scmurl=full_url,
)
db.session.add(build)
existing_build = models.ComponentBuild.query.filter_by(
module_id=module.id, package=pkgname).first()
if (existing_build
and existing_build.state != models.BUILD_STATES['done']):
existing_build.state = models.BUILD_STATES['init']
db.session.add(existing_build)
else:
# XXX: what about components that were present in previous
# builds but are gone now (component reduction)?
build = models.ComponentBuild(
module_id=module.id,
package=pkgname,
format="rpms",
scmurl=full_url,
)
db.session.add(build)
module.modulemd = mmd.dumps()
module.transition(conf, models.BUILD_STATES["wait"])

1
test-requirements.txt
View File

@@ -1,2 +1,3 @@
nose
mock
copr-backend

13
tests/test_views/fakemodule2.yaml Normal file
View File

@@ -0,0 +1,13 @@
document: modulemd
version: 0
data:
name: fakemodule2
version: 4.3.44
release: 5
summary: A fake module containing the bash shell
description: >
A fake module used for testing
license:
module:
- MIT
content: []

36
tests/test_views/test_views.py
View File

@@ -194,6 +194,42 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'wait')
@patch('module_build_service.auth.get_username', return_value='Homer J. Simpson')
@patch('module_build_service.auth.assert_is_packager')
@patch('module_build_service.scm.SCM')
def test_submit_componentless_build(self, mocked_scm, mocked_assert_is_packager,
mocked_get_username):
def mocked_scm_checkout(temp_dir):
scm_dir = path.join(temp_dir, 'fakemodule2')
mkdir(scm_dir)
base_dir = path.abspath(path.dirname(__file__))
copyfile(path.join(base_dir, 'fakemodule2.yaml'),
path.join(scm_dir, 'fakemodule2.yaml'))
return scm_dir
mocked_scm.return_value.checkout = mocked_scm_checkout
mocked_scm.return_value.name = 'fakemodule2'
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertEquals(data['component_builds'], [])
self.assertEquals(data['name'], 'fakemodule2')
self.assertEquals(data['scmurl'],
('git://pkgs.stg.fedoraproject.org/modules/testmodule'
'.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'))
self.assertEquals(data['release'], '5')
self.assertTrue(data['time_submitted'] is not None)
self.assertTrue(data['time_modified'] is not None)
self.assertEquals(data['release'], '5')
self.assertEquals(data['time_completed'], None)
self.assertEquals(data['version'], '4.3.44')
self.assertEquals(data['owner'], 'Homer J. Simpson')
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'wait')
def test_submit_build_cert_error(self):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'