From 0f797c291789f3fbe2a62bb7f96db5ed41f4a612 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Tue, 10 Jan 2017 15:58:26 +0100 Subject: [PATCH] Remove fake base-runtime module from DB and old resolve_profiles method because they are not used anymore. Fix the tests to work with remove fake base-runtime. --- module_build_service/manage.py | 3 -- module_build_service/models.py | 42 --------------- module_build_service/utils.py | 92 -------------------------------- tests/__init__.py | 7 +-- tests/test_models/test_models.py | 14 +---- tests/test_views/test_views.py | 69 ++++++++++++------------ 6 files changed, 38 insertions(+), 189 deletions(-) diff --git a/module_build_service/manage.py b/module_build_service/manage.py index 26a9627a..06fb7f3e 100644 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -42,7 +42,6 @@ from module_build_service.pdc import ( get_module_tag, get_module_build_dependencies) from module_build_service.utils import ( submit_module_build, - insert_fake_baseruntime, ) from module_build_service.messaging import RidaModule import module_build_service.messaging @@ -135,7 +134,6 @@ def upgradedb(): 'migrations') with app.app_context(): flask_migrate.upgrade(directory=migrations_dir) - insert_fake_baseruntime() @manager.command @@ -172,7 +170,6 @@ def build_module_locally(url): # In the future, we should use PDC to get what we need from the fake module, # so it's probably not big problem. db.create_all() - insert_fake_baseruntime() username = getpass.getuser() submit_module_build(username, url, allow_local_url=True) diff --git a/module_build_service/models.py b/module_build_service/models.py index 79126d9c..009fa88f 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -287,48 +287,6 @@ class ModuleBuild(RidaBase): return tasks - def resolve_profiles(self, session, key, seen=None): - """ Gather dependency profiles named `key` of modules we depend on. - - This is used to find the union of all 'buildroot' profiles of a - module's dependencies. - - https://pagure.io/fm-orchestrator/issue/181 - """ - - seen = seen or [] # Initialize to an empty list. - result = set() - for name, stream in self.mmd().buildrequires.items(): - # First, guard against infinite recursion - if name in seen: - continue - - # Find the latest of the dep in our db of built modules. - dep = session.query(ModuleBuild)\ - .filter(ModuleBuild.name==name)\ - .filter(ModuleBuild.stream==stream)\ - .filter(or_( - ModuleBuild.state==BUILD_STATES["done"], - ModuleBuild.state==BUILD_STATES["ready"], - )).order_by('version').first() - - # XXX - We may want to make this fatal one day, but warn for now. - if not dep: - log.warn("Could not find built dep " - "%s/%s for %r" % (name, stream, self)) - continue - - # Take note of what rpms are in this dep's profile. - profiles = dep.mmd().profiles - if key in profiles: - result |= profiles[key].rpms - - # And recurse to all modules that are deps of our dep. - result |= dep.resolve_profiles(session, key, seen + [name]) - - # Return the union of all rpms in all profiles of the given key. - return result - def __repr__(self): return "" % ( self.name, self.stream, self.version, diff --git a/module_build_service/utils.py b/module_build_service/utils.py index df298518..06fc274b 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -427,98 +427,6 @@ def submit_module_build(username, url, allow_local_url = False): mmd.name, mmd.stream, mmd.version) return module - -def insert_fake_baseruntime(): - """ Insert a fake base-runtime module into our db. - - This is done so that we can reference the build profiles in its modulemd. - - See: - - https://pagure.io/fm-orchestrator/pull-request/228 - - https://pagure.io/fm-orchestrator/pull-request/225 - """ - - import sqlalchemy as sa - - import modulemd - - yaml = """ - document: modulemd - version: 1 - data: - name: base-runtime - stream: master - version: 3 - summary: A fake base-runtime module, used to bootstrap the infrastructure. - description: ... - profiles: - buildroot: - rpms: - - bash - - bzip2 - - coreutils - - cpio - - diffutils - - fedora-release - - findutils - - gawk - - gcc - - gcc-c++ - - grep - - gzip - - info - - make - - patch - - redhat-rpm-config - - rpm-build - - sed - - shadow-utils - - tar - - unzip - - util-linux - - which - - xz - srpm-buildroot: - rpms: - - bash - - fedora-release - - fedpkg-minimal - - gnupg2 - - redhat-rpm-config - - rpm-build - - shadow-utils - """ - - mmd = modulemd.ModuleMetadata() - mmd.loads(yaml) - - # Check to see if this thing already exists... - query = models.ModuleBuild.query\ - .filter_by(name=mmd.name)\ - .filter_by(stream=mmd.stream)\ - .filter_by(version=mmd.version) - if query.count(): - logging.info('%r exists. Skipping creation.' % query.first()) - return - - # Otherwise, it does not exist. So, create it. - now = datetime.utcnow() - module = models.ModuleBuild() - module.name = mmd.name - module.stream = mmd.stream - module.version = mmd.version - module.modulemd = yaml - module.scmurl = '...' - module.owner = 'modularity' - module.state = models.BUILD_STATES['done'] - module.state_reason = 'Artificially created.' - module.time_submitted = now - module.time_modified = now - module.time_completed = now - db.session.add(module) - db.session.commit() - - def scm_url_schemes(terse=False): """ Definition of URL schemes supported by both frontend and scheduler. diff --git a/tests/__init__.py b/tests/__init__.py index 12c869b2..74481b30 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -27,7 +27,6 @@ from datetime import datetime, timedelta from module_build_service import db from module_build_service.config import init_config from module_build_service.models import ModuleBuild, ComponentBuild -from module_build_service.utils import insert_fake_baseruntime import module_build_service.pdc app = module_build_service.app @@ -61,7 +60,6 @@ def init_data(): db.session.remove() db.drop_all() db.create_all() - insert_fake_baseruntime() for index in range(10): build_one = ModuleBuild() build_one.name = 'nginx' @@ -208,7 +206,6 @@ def scheduler_init_data(): db.session.remove() db.drop_all() db.create_all() - insert_fake_baseruntime() current_dir = os.path.dirname(__file__) star_command_yml_path = os.path.join( @@ -241,7 +238,7 @@ def scheduler_init_data(): component_one_build_one.state = None component_one_build_one.nvr = 'communicator-1.10.1-2.module_starcommand_1_3' component_one_build_one.batch = 2 - component_one_build_one.module_id = 2 + component_one_build_one.module_id = 1 component_two_build_one = module_build_service.models.ComponentBuild() component_two_build_one.package = 'module-build-macros' @@ -254,7 +251,7 @@ def scheduler_init_data(): component_two_build_one.nvr = \ 'module-build-macros-01-1.module_starcommand_1_3' component_two_build_one.batch = 2 - component_two_build_one.module_id = 2 + component_two_build_one.module_id = 1 db.session.add(build_one) db.session.add(component_one_build_one) diff --git a/tests/test_models/test_models.py b/tests/test_models/test_models.py index 466831e9..bafee720 100644 --- a/tests/test_models/test_models.py +++ b/tests/test_models/test_models.py @@ -33,16 +33,4 @@ class TestModels(unittest.TestCase): def setUp(self): init_data() - def test_resolve_refs(self): - expected = set([ - 'shadow-utils', - 'fedora-release', - 'redhat-rpm-config', - 'rpm-build', - 'fedpkg-minimal', - 'gnupg2', - 'bash', - ]) - build = db.session.query(models.ModuleBuild).filter_by(name='testmodule').one() - result = build.resolve_profiles(db.session, 'srpm-buildroot') - eq_(result, expected) + diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index c29708c2..d468eca5 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -82,16 +82,17 @@ class TestViews(unittest.TestCase): init_data() def test_query_build(self): - rv = self.client.get('/module-build-service/1/module-builds/2') + rv = self.client.get('/module-build-service/1/module-builds/1') data = json.loads(rv.data) - self.assertEquals(data['id'], 2) + self.assertEquals(data['id'], 1) self.assertEquals(data['name'], 'nginx') self.assertEquals(data['owner'], 'Moe Szyslak') self.assertEquals(data['state'], 3) self.assertEquals(data['state_reason'], None) self.assertEquals(data['tasks'], { - 'rpms/module-build-macros': '47383993/1', - 'rpms/postgresql': '2433433/1'} + 'rpms/module-build-macros': '12312321/1', + 'rpms/nginx': '12312345/1' + } ) self.assertEquals(data['time_completed'], '2016-09-03T11:25:32Z') self.assertEquals(data['time_modified'], '2016-09-03T11:25:32Z') @@ -108,7 +109,7 @@ class TestViews(unittest.TestCase): 'module-build-service/1/module-builds/?per_page=8&page=4' in meta_data['last']) self.assertTrue( 'module-build-service/1/module-builds/?per_page=8&page=1' in meta_data['first']) - self.assertEquals(meta_data['total'], 31) + self.assertEquals(meta_data['total'], 30) self.assertEquals(meta_data['per_page'], 8) self.assertEquals(meta_data['pages'], 4) self.assertEquals(meta_data['page'], 2) @@ -121,14 +122,14 @@ class TestViews(unittest.TestCase): def test_query_builds_verbose(self): rv = self.client.get('/module-build-service/1/module-builds/?per_page=2&verbose=True') - item = json.loads(rv.data)['items'][1] - self.assertEquals(item['id'], 2) + item = json.loads(rv.data)['items'][0] + self.assertEquals(item['id'], 1) self.assertEquals(item['name'], 'nginx') self.assertEquals(item['owner'], 'Moe Szyslak') self.assertEquals(item['state'], 3) self.assertEquals(item['tasks'], { - 'rpms/module-build-macros': '47383993/1', - 'rpms/postgresql': '2433433/1' + 'rpms/module-build-macros': '12312321/1', + 'rpms/nginx': '12312345/1' } ) self.assertEquals(item['time_completed'], '2016-09-03T11:25:32Z') @@ -150,7 +151,7 @@ class TestViews(unittest.TestCase): rv = self.client.get( '/module-build-service/1/module-builds/?completed_after=2016-09-03T12:25:00Z') data = json.loads(rv.data) - self.assertEquals(data['meta']['total'], 9) + self.assertEquals(data['meta']['total'], 8) def test_query_builds_filter_submitted_before(self): rv = self.client.get( @@ -162,7 +163,7 @@ class TestViews(unittest.TestCase): rv = self.client.get( '/module-build-service/1/module-builds/?submitted_after=2016-09-03T12:25:00Z') data = json.loads(rv.data) - self.assertEquals(data['meta']['total'], 24) + self.assertEquals(data['meta']['total'], 23) def test_query_builds_filter_modified_before(self): rv = self.client.get( @@ -174,7 +175,7 @@ class TestViews(unittest.TestCase): rv = self.client.get( '/module-build-service/1/module-builds/?modified_after=2016-09-03T12:25:00Z') data = json.loads(rv.data) - self.assertEquals(data['meta']['total'], 25) + self.assertEquals(data['meta']['total'], 24) def test_query_builds_filter_owner(self): rv = self.client.get( @@ -186,7 +187,7 @@ class TestViews(unittest.TestCase): rv = self.client.get( '/module-build-service/1/module-builds/?state=3') data = json.loads(rv.data) - self.assertEquals(data['meta']['total'], 21) + self.assertEquals(data['meta']['total'], 20) def test_query_builds_two_filters(self): rv = self.client.get('/module-build-service/1/module-builds/?owner=Moe%20Szyslak' @@ -212,14 +213,14 @@ class TestViews(unittest.TestCase): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) data = json.loads(rv.data) self.assertEquals(data['component_builds'], [61]) self.assertEquals(data['name'], 'fakemodule') self.assertEquals(data['scmurl'], ('git://pkgs.stg.fedoraproject.org/modules/testmodule' - '.git?#68932c90de214d9d13feefbd35246a81b6cb8d49')) + '.git?#68931c90de214d9d13feefbd35246a81b6cb8d49')) self.assertEquals(data['version'], '5') self.assertTrue(data['time_submitted'] is not None) self.assertTrue(data['time_modified'] is not None) @@ -227,9 +228,9 @@ class TestViews(unittest.TestCase): self.assertEquals(data['time_completed'], None) self.assertEquals(data['stream'], '4.3.44') self.assertEquals(data['owner'], 'Homer J. Simpson') - self.assertEquals(data['id'], 32) + self.assertEquals(data['id'], 31) self.assertEquals(data['state_name'], 'wait') - self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/32') + self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31') mmd = _modulemd.ModuleMetadata() mmd.loads(data["modulemd"]) @@ -242,14 +243,14 @@ class TestViews(unittest.TestCase): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) data = json.loads(rv.data) self.assertEquals(data['component_builds'], []) self.assertEquals(data['name'], 'fakemodule2') self.assertEquals(data['scmurl'], ('git://pkgs.stg.fedoraproject.org/modules/testmodule' - '.git?#68932c90de214d9d13feefbd35246a81b6cb8d49')) + '.git?#68931c90de214d9d13feefbd35246a81b6cb8d49')) self.assertEquals(data['version'], '5') self.assertTrue(data['time_submitted'] is not None) self.assertTrue(data['time_modified'] is not None) @@ -257,13 +258,13 @@ class TestViews(unittest.TestCase): self.assertEquals(data['time_completed'], None) self.assertEquals(data['stream'], '4.3.44') self.assertEquals(data['owner'], 'Homer J. Simpson') - self.assertEquals(data['id'], 32) + self.assertEquals(data['id'], 31) self.assertEquals(data['state_name'], 'wait') def test_submit_build_auth_error(self): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#48932b90de214d9d13feefbd35246a81b6cb8d49'})) + 'testmodule.git?#48931b90de214d9d13feefbd35246a81b6cb8d49'})) data = json.loads(rv.data) self.assertEquals( data['message'], @@ -309,7 +310,7 @@ class TestViews(unittest.TestCase): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) data = json.loads(rv.data) self.assertTrue(data['message'].startswith('Invalid modulemd:')) self.assertEquals(data['status'], 422) @@ -330,19 +331,19 @@ class TestViews(unittest.TestCase): start = time.time() rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) data = json.loads(rv.data) self.assertEquals(len(data['component_builds']), 5) self.assertEquals(data['name'], 'base-runtime') self.assertEquals(data['scmurl'], ('git://pkgs.stg.fedoraproject.org/modules/testmodule' - '.git?#68932c90de214d9d13feefbd35246a81b6cb8d49')) + '.git?#68931c90de214d9d13feefbd35246a81b6cb8d49')) self.assertTrue(data['time_submitted'] is not None) self.assertTrue(data['time_modified'] is not None) self.assertEquals(data['time_completed'], None) self.assertEquals(data['owner'], 'Homer J. Simpson') - self.assertEquals(data['id'], 32) + self.assertEquals(data['id'], 31) self.assertEquals(data['state_name'], 'wait') # SCM availability check is parallelized, so 5 components should not @@ -364,7 +365,7 @@ class TestViews(unittest.TestCase): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) data = json.loads(rv.data) self.assertEquals(data['status'], 422) @@ -381,14 +382,14 @@ class TestViews(unittest.TestCase): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' - 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + 'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'})) data = json.loads(rv.data) self.assertEquals(data['component_builds'], [61, 62]) self.assertEquals(data['name'], 'fakemodule') self.assertEquals(data['scmurl'], ('git://pkgs.stg.fedoraproject.org/modules/testmodule' - '.git?#68932c90de214d9d13feefbd35246a81b6cb8d49')) + '.git?#68931c90de214d9d13feefbd35246a81b6cb8d49')) self.assertEquals(data['version'], '5') self.assertTrue(data['time_submitted'] is not None) self.assertTrue(data['time_modified'] is not None) @@ -396,12 +397,12 @@ class TestViews(unittest.TestCase): self.assertEquals(data['time_completed'], None) self.assertEquals(data['stream'], '4.3.44') self.assertEquals(data['owner'], 'Homer J. Simpson') - self.assertEquals(data['id'], 32) + self.assertEquals(data['id'], 31) self.assertEquals(data['state_name'], 'wait') - self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/32') + self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31') batches = {} - for build in ComponentBuild.query.filter_by(module_id=32).all(): + for build in ComponentBuild.query.filter_by(module_id=31).all(): batches[build.package] = build.batch self.assertEquals(batches["bash"], 2) @@ -411,7 +412,7 @@ class TestViews(unittest.TestCase): @patch('module_build_service.auth.assert_is_packager') def test_cancel_build(self, mocked_assert_is_packager, mocked_get_username): - rv = self.client.patch('/module-build-service/1/module-builds/31', + rv = self.client.patch('/module-build-service/1/module-builds/30', data=json.dumps({'state': 'failed'})) data = json.loads(rv.data) @@ -434,7 +435,7 @@ class TestViews(unittest.TestCase): @patch('module_build_service.auth.assert_is_packager') def test_cancel_build_wrong_param(self, mocked_assert_is_packager, mocked_get_username): - rv = self.client.patch('/module-build-service/1/module-builds/31', + rv = self.client.patch('/module-build-service/1/module-builds/30', data=json.dumps({'some_param': 'value'})) data = json.loads(rv.data) @@ -447,7 +448,7 @@ class TestViews(unittest.TestCase): @patch('module_build_service.auth.assert_is_packager') def test_cancel_build_wrong_state(self, mocked_assert_is_packager, mocked_get_username): - rv = self.client.patch('/module-build-service/1/module-builds/31', + rv = self.client.patch('/module-build-service/1/module-builds/30', data=json.dumps({'state': 'some_state'})) data = json.loads(rv.data)