modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-13 17:51:43 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add the ability to specify different rebuild methods

Browse Source
This commit is contained in:
mprahl
2017-10-26 15:36:08 -04:00
parent 6cf24baebb
commit 45cce653eb
10 changed files with 503 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/__init__.py
View File

@@ -47,11 +47,14 @@ def uncompress_vcrpy_cassette():
uncompress_vcrpy_cassette()
def init_data():
def clean_database():
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
def init_data():
clean_database()
for index in range(10):
build_one = ModuleBuild()
build_one.name = 'nginx'
@@ -72,6 +75,7 @@ def init_data():
datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10))
build_one.time_completed = \
datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10))
build_one.rebuild_strategy = 'changed-and-after'
component_one_build_one = ComponentBuild()
component_one_build_one.package = 'nginx'
@@ -115,6 +119,7 @@ def init_data():
datetime(2016, 9, 3, 12, 27, 19) + timedelta(minutes=(index * 10))
build_two.time_completed = \
datetime(2016, 9, 3, 11, 27, 19) + timedelta(minutes=(index * 10))
build_two.rebuild_strategy = 'changed-and-after'
component_one_build_two = ComponentBuild()
component_one_build_two.package = 'postgresql'
@@ -157,6 +162,7 @@ def init_data():
build_three.time_modified = \
datetime(2016, 9, 3, 12, 28, 40) + timedelta(minutes=(index * 10))
build_three.time_completed = None
build_three.rebuild_strategy = 'changed-and-after'
component_one_build_three = ComponentBuild()
component_one_build_three.package = 'rubygem-rails'
@@ -197,10 +203,7 @@ def init_data():
def scheduler_init_data(communicator_state=None):
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
clean_database()
current_dir = os.path.dirname(__file__)
star_command_yml_path = os.path.join(
@@ -222,6 +225,7 @@ def scheduler_init_data(communicator_state=None):
build_one.owner = 'Buzz Lightyear'
build_one.time_submitted = datetime(2016, 12, 9, 11, 23, 20)
build_one.time_modified = datetime(2016, 12, 9, 11, 25, 32)
build_one.rebuild_strategy = 'changed-and-after'
component_one_build_one = module_build_service.models.ComponentBuild()
component_one_build_one.package = 'communicator'
@@ -256,10 +260,7 @@ def scheduler_init_data(communicator_state=None):
def test_reuse_component_init_data():
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
clean_database()
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(
@@ -281,6 +282,7 @@ def test_reuse_component_init_data():
build_one.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
build_one.time_modified = datetime(2017, 2, 15, 16, 19, 35)
build_one.time_completed = datetime(2017, 2, 15, 16, 19, 35)
build_one.rebuild_strategy = 'changed-and-after'
component_one_build_one = module_build_service.models.ComponentBuild()
component_one_build_one.package = 'perl-Tangerine'
@@ -353,6 +355,7 @@ def test_reuse_component_init_data():
build_two.owner = 'Tom Brady'
build_two.time_submitted = datetime(2017, 2, 19, 16, 8, 18)
build_two.time_modified = datetime(2017, 2, 19, 16, 8, 18)
build_two.rebuild_strategy = 'changed-and-after'
component_one_build_two = module_build_service.models.ComponentBuild()
component_one_build_two.package = 'perl-Tangerine'
@@ -412,10 +415,7 @@ def test_reuse_component_init_data():
def test_reuse_shared_userspace_init_data():
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
clean_database()
with make_session(conf) as session:
mmd = modulemd.ModuleMetadata()
@@ -443,6 +443,7 @@ def test_reuse_shared_userspace_init_data():
build_one.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
build_one.time_modified = datetime(2017, 2, 15, 16, 19, 35)
build_one.time_completed = datetime(2017, 2, 15, 16, 19, 35)
build_one.rebuild_strategy = 'changed-and-after'
session.add(build_one)
@@ -490,6 +491,7 @@ def test_reuse_shared_userspace_init_data():
build_one.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
build_one.time_modified = datetime(2017, 2, 15, 16, 19, 35)
build_one.time_completed = datetime(2017, 2, 15, 16, 19, 35)
build_one.rebuild_strategy = 'changed-and-after'
session.add(build_one)

7
tests/test_models/__init__.py
View File

@@ -26,7 +26,7 @@ import module_build_service
import modulemd
from datetime import datetime
from module_build_service import db
from tests import db, clean_database
from module_build_service.config import init_config
from module_build_service.models import ModuleBuild, BUILD_STATES
@@ -53,13 +53,12 @@ def module_build_from_modulemd(yaml):
build.time_submitted = datetime(2016, 9, 3, 12, 28, 33)
build.time_modified = datetime(2016, 9, 3, 12, 28, 40)
build.time_completed = None
build.rebuild_strategy = 'changed-and-after'
return build
def init_data():
db.session.remove()
db.drop_all()
db.create_all()
clean_database()
for filename in os.listdir(datadir):
with open(datadir + filename, 'r') as f:
yaml = f.read()

7
tests/test_scheduler/test_module_init.py
View File

@@ -25,7 +25,7 @@ import unittest
from mock import patch, PropertyMock
import vcr
from tests import conf, db
from tests import conf, db, clean_database
from tests.test_views.test_views import FakeSCM
import module_build_service.messaging
import module_build_service.scheduler.handlers.modules
@@ -46,10 +46,7 @@ class TestModuleInit(unittest.TestCase):
with open(testmodule_yml_path, 'r') as f:
yaml = f.read()
scmurl = ('git://pkgs.domain.local/modules/testmodule?#da95886')
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
clean_database()
with make_session(conf) as session:
ModuleBuild.create(
session, conf, 'testmodule', '1', 3, yaml, scmurl, 'mprahl')

87
tests/test_utils/test_utils.py
View File

@@ -813,6 +813,93 @@ class TestBatches(unittest.TestCase):
self.assertIsNone(plc_component.reused_component_id)
mock_sbc.assert_called_once()
@patch('module_build_service.utils.start_build_component')
@patch('module_build_service.config.Config.rebuild_strategy',
new_callable=mock.PropertyMock, return_value='all')
def test_start_next_batch_build_rebuild_strategy_all(
self, mock_rm, mock_sbc, default_buildroot_groups):
"""
Tests that start_next_batch_build can't reuse any components in the batch because the
rebuild method is set to "all".
"""
module_build = models.ModuleBuild.query.filter_by(id=2).one()
module_build.rebuild_strategy = 'all'
module_build.batch = 1
builder = mock.MagicMock()
further_work = module_build_service.utils.start_next_batch_build(
conf, module_build, db.session, builder)
# Batch number should increase.
self.assertEqual(module_build.batch, 2)
# No component reuse messages should be returned
self.assertEqual(len(further_work), 0)
# Make sure that both components in the batch were submitted
self.assertEqual(len(mock_sbc.mock_calls), 2)
@patch('module_build_service.utils.start_build_component')
@patch('module_build_service.config.Config.rebuild_strategy',
new_callable=mock.PropertyMock, return_value='only-changed')
def test_start_next_batch_build_rebuild_strategy_only_changed(
self, mock_rm, mock_sbc, default_buildroot_groups):
"""
Tests that start_next_batch_build reuses all unchanged components in the batch because the
rebuild method is set to "only-changed". This means that one component is reused in batch
2, and even though the other component in batch 2 changed and was rebuilt, the component
in batch 3 can be reused.
"""
module_build = models.ModuleBuild.query.filter_by(id=2).one()
module_build.rebuild_strategy = 'only-changed'
module_build.batch = 1
# perl-List-Compare changed
plc_component = models.ComponentBuild.query.filter_by(
module_id=2, package='perl-List-Compare').one()
plc_component.ref = '5ceea46add2366d8b8c5a623a2fb563b625b9abd'
builder = mock.MagicMock()
further_work = module_build_service.utils.start_next_batch_build(
conf, module_build, db.session, builder)
# Batch number should increase
self.assertEqual(module_build.batch, 2)
# Make sure we only have one message returned for the one reused component
self.assertEqual(len(further_work), 1)
# The KojiBuildChange message in further_work should have build_new_state
# set to COMPLETE, but the current component build state in the DB should be set
# to BUILDING, so KojiBuildChange message handler handles the change
# properly.
self.assertEqual(further_work[0].build_new_state, koji.BUILD_STATES['COMPLETE'])
component_build = models.ComponentBuild.from_component_event(db.session, further_work[0])
self.assertEqual(component_build.state, koji.BUILD_STATES['BUILDING'])
self.assertEqual(component_build.package, 'perl-Tangerine')
self.assertIsNotNone(component_build.reused_component_id)
# Make sure perl-List-Compare is set to the build state as well but not reused
self.assertEqual(plc_component.state, koji.BUILD_STATES['BUILDING'])
self.assertIsNone(plc_component.reused_component_id)
mock_sbc.assert_called_once()
mock_sbc.reset_mock()
# Complete the build
plc_component.state = koji.BUILD_STATES['COMPLETE']
pt_component = models.ComponentBuild.query.filter_by(
module_id=2, package='perl-Tangerine').one()
pt_component.state = koji.BUILD_STATES['COMPLETE']
# Start the next build batch
further_work = module_build_service.utils.start_next_batch_build(
conf, module_build, db.session, builder)
# Batch number should increase
self.assertEqual(module_build.batch, 3)
# Verify that tangerine was reused even though perl-Tangerine was rebuilt in the previous
# batch
self.assertEqual(further_work[0].build_new_state, koji.BUILD_STATES['COMPLETE'])
component_build = models.ComponentBuild.from_component_event(db.session, further_work[0])
self.assertEqual(component_build.state, koji.BUILD_STATES['BUILDING'])
self.assertEqual(component_build.package, 'tangerine')
self.assertIsNotNone(component_build.reused_component_id)
mock_sbc.assert_not_called()
@patch('module_build_service.utils.start_build_component')
def test_start_next_batch_build_smart_scheduling(self, mock_sbc, default_buildroot_groups):
"""

159
tests/test_views/test_views.py
View File

@@ -153,6 +153,7 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['time_completed'], '2016-09-03T11:25:32Z')
self.assertEquals(data['time_modified'], '2016-09-03T11:25:32Z')
self.assertEquals(data['time_submitted'], '2016-09-03T11:23:20Z')
self.assertEqual(data['rebuild_strategy'], 'changed-and-after')
def test_query_build_with_verbose_mode(self):
rv = self.client.get('/module-build-service/1/module-builds/1?verbose=true')
@@ -195,6 +196,7 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['time_modified'], u'2016-09-03T11:25:32Z')
self.assertEquals(data['time_submitted'], u'2016-09-03T11:23:20Z')
self.assertEquals(data['version'], '2')
self.assertEqual(data['rebuild_strategy'], 'changed-and-after')
def test_pagination_metadata(self):
rv = self.client.get('/module-build-service/1/module-builds/?per_page=8&page=2')
@@ -231,6 +233,7 @@ class TestViews(unittest.TestCase):
'id': 30,
'koji_tag': None,
'name': 'testmodule',
'rebuild_strategy': 'changed-and-after',
'owner': 'some_other_user',
'scmurl': ('git://pkgs.domain.local/modules/testmodule?'
'#ca95886c7a443b36a9ce31abda1f9bef22f2f8c9'),
@@ -259,11 +262,12 @@ class TestViews(unittest.TestCase):
'time_submitted': '2016-09-03T13:58:33Z',
'version': '6'
},
{
{
'id': 29,
'koji_tag': 'module-postgressql-1.2',
'name': 'postgressql',
'owner': 'some_user',
'rebuild_strategy': 'changed-and-after',
'scmurl': ('git://pkgs.domain.local/modules/postgressql?'
'#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9'),
'state': 3,
@@ -492,6 +496,7 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['stream'], 'master')
self.assertEquals(data['owner'], 'Homer J. Simpson')
self.assertEquals(data['id'], 31)
self.assertEquals(data['rebuild_strategy'], 'changed-and-after')
self.assertEquals(data['state_name'], 'init')
self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31')
self.assertEquals(data['state_trace'], [])
@@ -499,6 +504,65 @@ class TestViews(unittest.TestCase):
mmd = _modulemd.ModuleMetadata()
mmd.loads(data["modulemd"])
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
@patch('module_build_service.config.Config.rebuild_strategy_allow_override',
new_callable=PropertyMock, return_value=True)
def test_submit_build_rebuild_strategy(self, mocked_rmao, mocked_scm, mocked_get_user):
FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'rebuild_strategy': 'only-changed',
'scmurl': ('git://pkgs.stg.fedoraproject.org/modules/testmodule.git?'
'#68931c90de214d9d13feefbd35246a81b6cb8d49')}))
data = json.loads(rv.data)
self.assertEquals(data['rebuild_strategy'], 'only-changed')
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
@patch('module_build_service.config.Config.rebuild_strategies_allowed',
new_callable=PropertyMock, return_value=['all'])
@patch('module_build_service.config.Config.rebuild_strategy_allow_override',
new_callable=PropertyMock, return_value=True)
def test_submit_build_rebuild_strategy_not_allowed(self, mock_rsao, mock_rsa, mocked_scm,
mocked_get_user):
FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'rebuild_strategy': 'only-changed',
'scmurl': ('git://pkgs.stg.fedoraproject.org/modules/testmodule.git?'
'#68931c90de214d9d13feefbd35246a81b6cb8d49')}))
data = json.loads(rv.data)
self.assertEqual(rv.status_code, 400)
expected_error = {
'error': 'Bad Request',
'message': ('The rebuild method of "only-changed" is not allowed. Chose from: all.'),
'status': 400
}
self.assertEqual(data, expected_error)
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_build_rebuild_strategy_override_not_allowed(self, mocked_scm, mocked_get_user):
FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'branch': 'master', 'rebuild_strategy': 'only-changed',
'scmurl': ('git://pkgs.stg.fedoraproject.org/modules/testmodule.git?'
'#68931c90de214d9d13feefbd35246a81b6cb8d49')}))
data = json.loads(rv.data)
self.assertEqual(rv.status_code, 400)
expected_error = {
'error': 'Bad Request',
'message': ('The request contains the "rebuild_strategy" parameter but overriding '
'the default isn\'t allowed'),
'status': 400
}
self.assertEqual(data, expected_error)
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_componentless_build(self, mocked_scm, mocked_get_user):
@@ -523,6 +587,7 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['owner'], 'Homer J. Simpson')
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'init')
self.assertEquals(data['rebuild_strategy'], 'changed-and-after')
def test_submit_build_auth_error(self):
base_dir = path.abspath(path.dirname(__file__))
@@ -820,3 +885,95 @@ class TestViews(unittest.TestCase):
data = json.loads(rv.data)
self.assertEqual(rv.status_code, 200)
self.assertEquals(data, {'auth_method': 'kerberos', 'version': version})
def test_rebuild_strategy_api(self):
rv = self.client.get('/module-build-service/1/rebuild-strategies/')
data = json.loads(rv.data)
self.assertEqual(rv.status_code, 200)
expected = {
'items': [
{
'allowed': False,
'default': False,
'description': 'All components will be rebuilt',
'name': 'all'
},
{
'allowed': True,
'default': True,
'description': ('All components that have changed and those in subsequent '
'batches will be rebuilt'),
'name': 'changed-and-after'
},
{
'allowed': False,
'default': False,
'description': 'All changed components will be rebuilt',
'name': 'only-changed'
}
]
}
self.assertEquals(data, expected)
def test_rebuild_strategy_api_only_changed_default(self):
with patch.object(mbs_config.Config, 'rebuild_strategy', new_callable=PropertyMock) as r_s:
r_s.return_value = 'only-changed'
rv = self.client.get('/module-build-service/1/rebuild-strategies/')
data = json.loads(rv.data)
self.assertEqual(rv.status_code, 200)
expected = {
'items': [
{
'allowed': False,
'default': False,
'description': 'All components will be rebuilt',
'name': 'all'
},
{
'allowed': False,
'default': False,
'description': ('All components that have changed and those in subsequent '
'batches will be rebuilt'),
'name': 'changed-and-after'
},
{
'allowed': True,
'default': True,
'description': 'All changed components will be rebuilt',
'name': 'only-changed'
}
]
}
self.assertEquals(data, expected)
def test_rebuild_strategy_api_override_allowed(self):
with patch.object(mbs_config.Config, 'rebuild_strategy_allow_override',
new_callable=PropertyMock) as rsao:
rsao.return_value = True
rv = self.client.get('/module-build-service/1/rebuild-strategies/')
data = json.loads(rv.data)
self.assertEqual(rv.status_code, 200)
expected = {
'items': [
{
'allowed': True,
'default': False,
'description': 'All components will be rebuilt',
'name': 'all'
},
{
'allowed': True,
'default': True,
'description': ('All components that have changed and those in subsequent '
'batches will be rebuilt'),
'name': 'changed-and-after'
},
{
'allowed': True,
'default': False,
'description': 'All changed components will be rebuilt',
'name': 'only-changed'
}
]
}
self.assertEquals(data, expected)