diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 3f0d3a4e..aaf8e5ee 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -215,8 +215,8 @@ class TestModuleBuilder(GenericBuilder): pass -@patch("module_build_service.config.Config.system", - new_callable=PropertyMock, return_value="test") +@patch.object(module_build_service.config.Config, 'system', new_callable=PropertyMock, + return_value='test') @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={ 'srpm-build': @@ -292,7 +292,7 @@ class TestBuild(unittest.TestCase): # Check that the components are added to buildroot after the batch # is built. buildroot_groups = [] - buildroot_groups.append(set([u'module-build-macros-0.1-1.module+fc4ed5f7.src.rpm-1-1'])) + buildroot_groups.append(set([u'module-build-macros-0.1-1.module+24957a32.src.rpm-1-1'])) buildroot_groups.append(set([u'perl-Tangerine?#f24-1-1', u'perl-List-Compare?#f25-1-1'])) buildroot_groups.append(set([u'tangerine?#f23-1-1'])) @@ -318,31 +318,45 @@ class TestBuild(unittest.TestCase): @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') - def test_submit_build_from_yaml(self, mocked_scm, mocked_get_user, conf_system, dbg): + def test_submit_build_from_yaml_not_allowed( + self, mocked_scm, mocked_get_user, conf_system, dbg): MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") testmodule = os.path.join(base_dir, 'staged_data', 'testmodule.yaml') with open(testmodule) as f: yaml = f.read() - def submit(): + with patch.object(module_build_service.config.Config, 'yaml_submit_allowed', + new_callable=PropertyMock, return_value=False): rv = self.client.post('/module-build-service/1/module-builds/', content_type='multipart/form-data', data={'yaml': (testmodule, yaml)}) - return json.loads(rv.data) - - with patch("module_build_service.config.Config.yaml_submit_allowed", - new_callable=PropertyMock, return_value=True): - conf.set_item("yaml_submit_allowed", True) - data = submit() - self.assertEqual(data['id'], 1) - - with patch("module_build_service.config.Config.yaml_submit_allowed", - new_callable=PropertyMock, return_value=False): - data = submit() + data = json.loads(rv.data) self.assertEqual(data['status'], 403) self.assertEqual(data['message'], 'YAML submission is not enabled') + @timed(30) + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + def test_submit_build_from_yaml_allowed(self, mocked_scm, mocked_get_user, conf_system, dbg): + MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") + + testmodule = os.path.join(base_dir, 'staged_data', 'testmodule.yaml') + with open(testmodule) as f: + yaml = f.read() + + with patch.object(module_build_service.config.Config, 'yaml_submit_allowed', + new_callable=PropertyMock, return_value=True): + rv = self.client.post('/module-build-service/1/module-builds/', + content_type='multipart/form-data', + data={'yaml': (testmodule, yaml)}) + data = json.loads(rv.data) + self.assertEqual(data['id'], 1) + + msgs = [] + stop = module_build_service.scheduler.make_simple_stop_condition(db.session) + module_build_service.scheduler.main(msgs, stop) + @timed(30) @patch('module_build_service.auth.get_user', return_value=user) def test_submit_build_with_optional_params(self, mocked_get_user, conf_system, dbg): @@ -874,4 +888,3 @@ class TestLocalBuild(unittest.TestCase): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) self.assertTrue(build.module_build.state in [ models.BUILD_STATES["done"], models.BUILD_STATES["ready"]]) - diff --git a/tests/test_scheduler/test_module_init.py b/tests/test_scheduler/test_module_init.py new file mode 100644 index 00000000..d796caf5 --- /dev/null +++ b/tests/test_scheduler/test_module_init.py @@ -0,0 +1,153 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import os +import unittest + +from mock import patch, PropertyMock +import vcr + +from tests import conf, db +from tests.test_views.test_views import MockedSCM +import module_build_service.messaging +import module_build_service.scheduler.handlers.modules +from module_build_service import build_logs +from module_build_service.models import make_session, ModuleBuild, ComponentBuild + +CASSETTE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'vcr-request-data/') + + +class TestModuleInit(unittest.TestCase): + + def setUp(self): + self.fn = module_build_service.scheduler.handlers.modules.init + self.staged_data_dir = os.path.join( + os.path.dirname(__file__), '../', 'staged_data') + testmodule_yml_path = os.path.join( + self.staged_data_dir, 'testmodule.yaml') + with open(testmodule_yml_path, 'r') as f: + yaml = f.read() + scmurl = ('git://pkgs.domain.local/modules/testmodule?#da95886') + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + with make_session(conf) as session: + ModuleBuild.create( + session, conf, 'testmodule', '1', 3, yaml, scmurl, 'mprahl') + + filename = os.path.join(CASSETTE_DIR, self.id()) + self.vcr = vcr.use_cassette(filename) + self.vcr.__enter__() + + def tearDown(self): + self.vcr.__exit__() + try: + path = build_logs.path(1) + os.remove(path) + except: + pass + + @patch('module_build_service.scm.SCM') + def test_init_basic(self, mocked_scm): + MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + msg = module_build_service.messaging.MBSModule( + msg_id=None, module_build_id=1, module_build_state='init') + with make_session(conf) as session: + self.fn(config=conf, session=session, msg=msg) + build = ModuleBuild.query.filter_by(id=1).one() + # Make sure the module entered the wait state + assert build.state == 1, build.state + # Make sure format_mmd was run properly + assert type(build.mmd().xmd['mbs']) is dict + + @patch('module_build_service.scm.SCM') + def test_init_scm_not_available(self, mocked_scm): + def mocked_scm_get_latest(): + raise RuntimeError("Failed in mocked_scm_get_latest") + + MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + mocked_scm.return_value.get_latest = mocked_scm_get_latest + msg = module_build_service.messaging.MBSModule( + msg_id=None, module_build_id=1, module_build_state='init') + with make_session(conf) as session: + self.fn(config=conf, session=session, msg=msg) + build = ModuleBuild.query.filter_by(id=1).one() + # Make sure the module entered the failed state + # since the git server is not available + assert build.state == 4, build.state + + @patch("module_build_service.config.Config.modules_allow_repository", + new_callable=PropertyMock, return_value=True) + @patch('module_build_service.scm.SCM') + def test_init_includedmodule(self, mocked_scm, mocked_mod_allow_repo): + MockedSCM(mocked_scm, "includedmodules", ['testmodule.yaml']) + includedmodules_yml_path = os.path.join( + self.staged_data_dir, 'includedmodules.yaml') + with open(includedmodules_yml_path, 'r') as f: + yaml = f.read() + scmurl = ('git://pkgs.domain.local/modules/includedmodule?#da95886') + with make_session(conf) as session: + ModuleBuild.create( + session, conf, 'includemodule', '1', 3, yaml, scmurl, 'mprahl') + msg = module_build_service.messaging.MBSModule( + msg_id=None, module_build_id=2, module_build_state='init') + self.fn(config=conf, session=session, msg=msg) + build = ModuleBuild.query.filter_by(id=2).one() + assert build.state == 1 + assert build.name == 'includemodule' + batches = {} + for comp_build in ComponentBuild.query.filter_by(module_id=2).all(): + batches[comp_build.package] = comp_build.batch + assert batches['ed'] == 2 + assert batches['perl-List-Compare'] == 2 + assert batches['perl-Tangerine'] == 2 + assert batches['tangerine'] == 3 + assert batches['file'] == 4 + # Test that the RPMs are properly merged in xmd + xmd_rpms = { + 'ed': {'ref': '40bd001563'}, + 'perl-List-Compare': {'ref': '2ee8474e44'}, + 'tangerine': {'ref': 'd29d5c24b8'}, + 'file': {'ref': 'a2740663f8'}, + 'perl-Tangerine': {'ref': '27785f9f05'} + } + assert build.mmd().xmd['mbs']['rpms'] == xmd_rpms + + @patch('module_build_service.models.ModuleBuild.from_module_event') + @patch('module_build_service.scm.SCM') + def test_init_when_get_latest_raises(self, mocked_scm, mocked_from_module_event): + MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', + '7035bd33614972ac66559ac1fdd019ff6027ad22', + get_latest_raise=True) + msg = module_build_service.messaging.MBSModule( + msg_id=None, module_build_id=1, module_build_state='init') + with make_session(conf) as session: + build = session.query(ModuleBuild).filter_by(id=1).one() + mocked_from_module_event.return_value = build + self.fn(config=conf, session=session, msg=msg) + # Query the database again to make sure the build object is updated + session.refresh(build) + # Make sure the module entered the failed state + assert build.state == 4, build.state + assert 'Failed to validate modulemd file' in build.state_reason diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 761e9e97..0cfd38e5 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -603,15 +603,17 @@ class TestUtils(unittest.TestCase): "Tom Brady", 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#8fea453', 'master') - self.assertEqual(module_build.state, models.BUILD_STATES['wait']) + self.assertEqual(module_build.state, models.BUILD_STATES['init']) self.assertEqual(module_build.batch, 0) self.assertEqual(module_build.state_reason, "Resubmitted by Tom Brady") self.assertEqual(complete_component.state, koji.BUILD_STATES['COMPLETE']) - self.assertEqual(failed_component.state, None) - self.assertEqual(canceled_component.state, None) + # These are still cancelled and failed until the init handler runs + self.assertEqual(failed_component.state, koji.BUILD_STATES['FAILED']) + self.assertEqual(canceled_component.state, koji.BUILD_STATES['CANCELED']) @vcr.use_cassette( - path.join(CASSETTES_DIR, 'tests.test_utils.TestUtils.test_format_mmd')) + path.join(CASSETTES_DIR, ('tests.test_utils.TestUtils.' + 'test_record_component_builds_duplicate_components'))) @patch('module_build_service.scm.SCM') def test_record_component_builds_duplicate_components(self, mocked_scm): with app.app_context(): diff --git a/tests/test_views/__init__.py b/tests/test_views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 104b3e50..0a9c34fa 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -440,7 +440,7 @@ class TestViews(unittest.TestCase): data = json.loads(rv.data) assert 'component_builds' in data, data - self.assertEquals(data['component_builds'], [61, 62, 63]) + self.assertEquals(data['component_builds'], []) self.assertEquals(data['name'], 'testmodule') self.assertEquals(data['scmurl'], ('git://pkgs.stg.fedoraproject.org/modules/testmodule' @@ -452,34 +452,10 @@ class TestViews(unittest.TestCase): self.assertEquals(data['stream'], 'master') self.assertEquals(data['owner'], 'Homer J. Simpson') self.assertEquals(data['id'], 31) - self.assertEquals(data['state_name'], 'wait') + self.assertEquals(data['state_name'], 'init') self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31') - self.assertEquals(data['state_trace'][0]['reason'], None) - self.assertTrue(data['state_trace'][0]['time'] is not None) - self.assertEquals(data['state_trace'][0]['state'], 1) - self.assertEquals(data['state_trace'][0]['state_name'], 'wait') - self.assertDictEqual(data['tasks'], { - 'rpms': { - 'perl-List-Compare': { - 'task_id': None, - 'state': None, - 'state_reason': None, - 'nvr': None, - }, - 'perl-Tangerine': { - 'task_id': None, - 'state': None, - 'state_reason': None, - 'nvr': None, - }, - 'tangerine': { - 'task_id': None, - 'state': None, - 'state_reason': None, - 'nvr': None, - }, - }, - }) + self.assertEquals(data['state_trace'], []) + self.assertDictEqual(data['tasks'], {}) mmd = _modulemd.ModuleMetadata() mmd.loads(data["modulemd"]) @@ -506,7 +482,7 @@ class TestViews(unittest.TestCase): self.assertEquals(data['stream'], 'master') self.assertEquals(data['owner'], 'Homer J. Simpson') self.assertEquals(data['id'], 31) - self.assertEquals(data['state_name'], 'wait') + self.assertEquals(data['state_name'], 'init') def test_submit_build_auth_error(self): base_dir = path.abspath(path.dirname(__file__)) @@ -558,112 +534,6 @@ class TestViews(unittest.TestCase): self.assertEquals(data['status'], 422) self.assertEquals(data['error'], 'Unprocessable Entity') - @patch('module_build_service.auth.get_user', return_value=user) - @patch('module_build_service.scm.SCM') - def test_submit_build_scm_parallalization(self, mocked_scm, - mocked_get_user): - def mocked_scm_get_latest(branch="master"): - time.sleep(1) - return branch - - MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', - '620ec77321b2ea7b0d67d82992dda3e1d67055b4') - mocked_scm.return_value.is_available = mocked_scm_get_latest - - start = time.time() - rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( - {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) - data = json.loads(rv.data) - - self.assertEquals(len(data['component_builds']), 3) - self.assertEquals(data['name'], 'testmodule') - self.assertEquals(data['scmurl'], - ('git://pkgs.stg.fedoraproject.org/modules/testmodule' - '.git?#68931c90de214d9d13feefbd35246a81b6cb8d49')) - self.assertTrue(data['time_submitted'] is not None) - self.assertTrue(data['time_modified'] is not None) - self.assertEquals(data['time_completed'], None) - self.assertEquals(data['owner'], 'Homer J. Simpson') - self.assertEquals(data['id'], 31) - self.assertEquals(data['state_name'], 'wait') - - # SCM availability check is parallelized, so 5 components should not - # take longer than 3 second, because each takes 1 second, but they - # are execute in 10 threads. They should take around 1 or 2 seconds - # max to complete. - self.assertTrue(time.time() - start < 3) - - @patch('module_build_service.auth.get_user', return_value=user) - @patch('module_build_service.scm.SCM') - def test_submit_build_scm_non_available(self, mocked_scm, mocked_get_user): - - def mocked_scm_get_latest(): - raise RuntimeError("Failed in mocked_scm_get_latest") - - MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', - '620ec77321b2ea7b0d67d82992dda3e1d67055b4') - mocked_scm.return_value.get_latest = mocked_scm_get_latest - - rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( - {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) - data = json.loads(rv.data) - - self.assertEquals(data['status'], 422) - self.assertEquals(data['message'][:31], "Failed to get the latest commit") - self.assertEquals(data['error'], "Unprocessable Entity") - - @patch('module_build_service.auth.get_user', return_value=user) - @patch('module_build_service.scm.SCM') - @patch("module_build_service.config.Config.modules_allow_repository", - new_callable=PropertyMock, return_value=True) - def test_submit_build_includedmodule(self, conf, mocked_scm, mocked_get_user): - MockedSCM(mocked_scm, "includedmodules", ["includedmodules.yaml", - "testmodule.yaml"]) - - rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( - {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) - data = json.loads(rv.data) - - assert 'component_builds' in data, data - self.assertEquals(data['component_builds'], [61, 62, 63, 64, 65]) - self.assertEquals(data['name'], 'includedmodules') - self.assertEquals(data['scmurl'], - ('git://pkgs.stg.fedoraproject.org/modules/testmodule' - '.git?#68931c90de214d9d13feefbd35246a81b6cb8d49')) - self.assertEquals(data['version'], '1') - self.assertTrue(data['time_submitted'] is not None) - self.assertTrue(data['time_modified'] is not None) - self.assertEquals(data['time_completed'], None) - self.assertEquals(data['stream'], 'master') - self.assertEquals(data['owner'], 'Homer J. Simpson') - self.assertEquals(data['id'], 31) - self.assertEquals(data['state_name'], 'wait') - self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31') - - batches = {} - for build in ComponentBuild.query.filter_by(module_id=31).all(): - batches[build.package] = build.batch - - self.assertEquals(batches['ed'], 2) - self.assertEquals(batches['perl-List-Compare'], 2) - self.assertEquals(batches['perl-Tangerine'], 2) - self.assertEquals(batches['tangerine'], 3) - self.assertEquals(batches["file"], 4) - - build = ModuleBuild.query.filter(ModuleBuild.id == data['id']).one() - mmd = build.mmd() - - # Test that RPMs are properly merged in case of included modules in mmd. - xmd_rpms = {'ed': {'ref': '40bd001563'}, - 'perl-List-Compare': {'ref': '2ee8474e44'}, - 'tangerine': {'ref': 'd29d5c24b8'}, - 'file': {'ref': 'a2740663f8'}, - 'perl-Tangerine': {'ref': '27785f9f05'}} - self.assertEqual(mmd.xmd['mbs']['rpms'], xmd_rpms) - @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') def test_submit_build_includedmodule_custom_repo_not_allowed(self, @@ -880,36 +750,6 @@ class TestViews(unittest.TestCase): self.assertEquals(data['status'], 422) self.assertEquals(data['error'], 'Unprocessable Entity') - @patch('module_build_service.auth.get_user', return_value=user) - @patch('module_build_service.scm.SCM') - @patch('module_build_service.models.ModuleBuild.from_module_event') - def test_submit_build_get_latest_raises( - self, from_module_event, mocked_scm, mocked_get_user): - MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', - '7035bd33614972ac66559ac1fdd019ff6027ad22', get_latest_raise=True) - - with app.app_context(): - rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( - {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#7035bd33614972ac66559ac1fdd019ff6027ad22'})) - data = json.loads(rv.data) - self.assertEquals(data['status'], 422) - self.assertEquals(data['error'], 'Unprocessable Entity') - - # Get the last module and check it has the state_reason set. - build = ModuleBuild.query.order_by(ModuleBuild.id).all()[-1] - db.session.add(build) - from_module_event.return_value = build - self.assertIn("Failed to validate modulemd file", build.state_reason) - - # Check that after the failed message is handled, the state_reason - # remains the same. - module_build_service.scheduler.handlers.modules.failed( - conf, db.session, MagicMock()) - db.session.expunge(build) - build = ModuleBuild.query.order_by(ModuleBuild.id).all()[-1] - self.assertIn("Failed to validate modulemd file", build.state_reason) - @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') @patch("module_build_service.config.Config.allow_custom_scmurls", new_callable=PropertyMock) diff --git a/tests/vcr-request-data.tar.gz b/tests/vcr-request-data.tar.gz index 122d31b5..88ce0960 100644 Binary files a/tests/vcr-request-data.tar.gz and b/tests/vcr-request-data.tar.gz differ