modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-04 19:28:49 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove fake base-runtime module from DB and old resolve_profiles method because they are not used anymore. Fix the tests to work with remove fake base-runtime.

Browse Source
This commit is contained in:
Jan Kaluza
2017-01-10 15:58:26 +01:00
parent dda47acab7
commit 0f797c2917
6 changed files with 38 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
module_build_service/manage.py
View File

@@ -42,7 +42,6 @@ from module_build_service.pdc import (
get_module_tag, get_module_build_dependencies)
from module_build_service.utils import (
submit_module_build,
insert_fake_baseruntime,
)
from module_build_service.messaging import RidaModule
import module_build_service.messaging
@@ -135,7 +134,6 @@ def upgradedb():
'migrations')
with app.app_context():
flask_migrate.upgrade(directory=migrations_dir)
insert_fake_baseruntime()
@manager.command
@@ -172,7 +170,6 @@ def build_module_locally(url):
# In the future, we should use PDC to get what we need from the fake module,
# so it's probably not big problem.
db.create_all()
insert_fake_baseruntime()
username = getpass.getuser()
submit_module_build(username, url, allow_local_url=True)

42
module_build_service/models.py
View File

@@ -287,48 +287,6 @@ class ModuleBuild(RidaBase):
return tasks
def resolve_profiles(self, session, key, seen=None):
""" Gather dependency profiles named `key` of modules we depend on.
This is used to find the union of all 'buildroot' profiles of a
module's dependencies.
https://pagure.io/fm-orchestrator/issue/181
"""
seen = seen or [] # Initialize to an empty list.
result = set()
for name, stream in self.mmd().buildrequires.items():
# First, guard against infinite recursion
if name in seen:
continue
# Find the latest of the dep in our db of built modules.
dep = session.query(ModuleBuild)\
.filter(ModuleBuild.name==name)\
.filter(ModuleBuild.stream==stream)\
.filter(or_(
ModuleBuild.state==BUILD_STATES["done"],
ModuleBuild.state==BUILD_STATES["ready"],
)).order_by('version').first()
# XXX - We may want to make this fatal one day, but warn for now.
if not dep:
log.warn("Could not find built dep "
"%s/%s for %r" % (name, stream, self))
continue
# Take note of what rpms are in this dep's profile.
profiles = dep.mmd().profiles
if key in profiles:
result |= profiles[key].rpms
# And recurse to all modules that are deps of our dep.
result |= dep.resolve_profiles(session, key, seen + [name])
# Return the union of all rpms in all profiles of the given key.
return result
def __repr__(self):
return "<ModuleBuild %s, stream=%s, version=%s, state %r, batch %r, state_reason %r>" % (
self.name, self.stream, self.version,

92
module_build_service/utils.py
View File

@@ -427,98 +427,6 @@ def submit_module_build(username, url, allow_local_url = False):
mmd.name, mmd.stream, mmd.version)
return module
def insert_fake_baseruntime():
""" Insert a fake base-runtime module into our db.
This is done so that we can reference the build profiles in its modulemd.
See:
- https://pagure.io/fm-orchestrator/pull-request/228
- https://pagure.io/fm-orchestrator/pull-request/225
"""
import sqlalchemy as sa
import modulemd
yaml = """
document: modulemd
version: 1
data:
name: base-runtime
stream: master
version: 3
summary: A fake base-runtime module, used to bootstrap the infrastructure.
description: ...
profiles:
buildroot:
rpms:
- bash
- bzip2
- coreutils
- cpio
- diffutils
- fedora-release
- findutils
- gawk
- gcc
- gcc-c++
- grep
- gzip
- info
- make
- patch
- redhat-rpm-config
- rpm-build
- sed
- shadow-utils
- tar
- unzip
- util-linux
- which
- xz
srpm-buildroot:
rpms:
- bash
- fedora-release
- fedpkg-minimal
- gnupg2
- redhat-rpm-config
- rpm-build
- shadow-utils
"""
mmd = modulemd.ModuleMetadata()
mmd.loads(yaml)
# Check to see if this thing already exists...
query = models.ModuleBuild.query\
.filter_by(name=mmd.name)\
.filter_by(stream=mmd.stream)\
.filter_by(version=mmd.version)
if query.count():
logging.info('%r exists. Skipping creation.' % query.first())
return
# Otherwise, it does not exist. So, create it.
now = datetime.utcnow()
module = models.ModuleBuild()
module.name = mmd.name
module.stream = mmd.stream
module.version = mmd.version
module.modulemd = yaml
module.scmurl = '...'
module.owner = 'modularity'
module.state = models.BUILD_STATES['done']
module.state_reason = 'Artificially created.'
module.time_submitted = now
module.time_modified = now
module.time_completed = now
db.session.add(module)
db.session.commit()
def scm_url_schemes(terse=False):
"""
Definition of URL schemes supported by both frontend and scheduler.

7
tests/__init__.py
View File

@@ -27,7 +27,6 @@ from datetime import datetime, timedelta
from module_build_service import db
from module_build_service.config import init_config
from module_build_service.models import ModuleBuild, ComponentBuild
from module_build_service.utils import insert_fake_baseruntime
import module_build_service.pdc
app = module_build_service.app
@@ -61,7 +60,6 @@ def init_data():
db.session.remove()
db.drop_all()
db.create_all()
insert_fake_baseruntime()
for index in range(10):
build_one = ModuleBuild()
build_one.name = 'nginx'
@@ -208,7 +206,6 @@ def scheduler_init_data():
db.session.remove()
db.drop_all()
db.create_all()
insert_fake_baseruntime()
current_dir = os.path.dirname(__file__)
star_command_yml_path = os.path.join(
@@ -241,7 +238,7 @@ def scheduler_init_data():
component_one_build_one.state = None
component_one_build_one.nvr = 'communicator-1.10.1-2.module_starcommand_1_3'
component_one_build_one.batch = 2
component_one_build_one.module_id = 2
component_one_build_one.module_id = 1
component_two_build_one = module_build_service.models.ComponentBuild()
component_two_build_one.package = 'module-build-macros'
@@ -254,7 +251,7 @@ def scheduler_init_data():
component_two_build_one.nvr = \
'module-build-macros-01-1.module_starcommand_1_3'
component_two_build_one.batch = 2
component_two_build_one.module_id = 2
component_two_build_one.module_id = 1
db.session.add(build_one)
db.session.add(component_one_build_one)

14
tests/test_models/test_models.py
View File

@@ -33,16 +33,4 @@ class TestModels(unittest.TestCase):
def setUp(self):
init_data()
def test_resolve_refs(self):
expected = set([
'shadow-utils',
'fedora-release',
'redhat-rpm-config',
'rpm-build',
'fedpkg-minimal',
'gnupg2',
'bash',
])
build = db.session.query(models.ModuleBuild).filter_by(name='testmodule').one()
result = build.resolve_profiles(db.session, 'srpm-buildroot')
eq_(result, expected)

69
tests/test_views/test_views.py
View File

@@ -82,16 +82,17 @@ class TestViews(unittest.TestCase):
init_data()
def test_query_build(self):
rv = self.client.get('/module-build-service/1/module-builds/2')
rv = self.client.get('/module-build-service/1/module-builds/1')
data = json.loads(rv.data)
self.assertEquals(data['id'], 2)
self.assertEquals(data['id'], 1)
self.assertEquals(data['name'], 'nginx')
self.assertEquals(data['owner'], 'Moe Szyslak')
self.assertEquals(data['state'], 3)
self.assertEquals(data['state_reason'], None)
self.assertEquals(data['tasks'], {
'rpms/module-build-macros': '47383993/1',
'rpms/postgresql': '2433433/1'}
'rpms/module-build-macros': '12312321/1',
'rpms/nginx': '12312345/1'
}
)
self.assertEquals(data['time_completed'], '2016-09-03T11:25:32Z')
self.assertEquals(data['time_modified'], '2016-09-03T11:25:32Z')
@@ -108,7 +109,7 @@ class TestViews(unittest.TestCase):
'module-build-service/1/module-builds/?per_page=8&page=4' in meta_data['last'])
self.assertTrue(
'module-build-service/1/module-builds/?per_page=8&page=1' in meta_data['first'])
self.assertEquals(meta_data['total'], 31)
self.assertEquals(meta_data['total'], 30)
self.assertEquals(meta_data['per_page'], 8)
self.assertEquals(meta_data['pages'], 4)
self.assertEquals(meta_data['page'], 2)
@@ -121,14 +122,14 @@ class TestViews(unittest.TestCase):
def test_query_builds_verbose(self):
rv = self.client.get('/module-build-service/1/module-builds/?per_page=2&verbose=True')
item = json.loads(rv.data)['items'][1]
self.assertEquals(item['id'], 2)
item = json.loads(rv.data)['items'][0]
self.assertEquals(item['id'], 1)
self.assertEquals(item['name'], 'nginx')
self.assertEquals(item['owner'], 'Moe Szyslak')
self.assertEquals(item['state'], 3)
self.assertEquals(item['tasks'], {
'rpms/module-build-macros': '47383993/1',
'rpms/postgresql': '2433433/1'
'rpms/module-build-macros': '12312321/1',
'rpms/nginx': '12312345/1'
}
)
self.assertEquals(item['time_completed'], '2016-09-03T11:25:32Z')
@@ -150,7 +151,7 @@ class TestViews(unittest.TestCase):
rv = self.client.get(
'/module-build-service/1/module-builds/?completed_after=2016-09-03T12:25:00Z')
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], 9)
self.assertEquals(data['meta']['total'], 8)
def test_query_builds_filter_submitted_before(self):
rv = self.client.get(
@@ -162,7 +163,7 @@ class TestViews(unittest.TestCase):
rv = self.client.get(
'/module-build-service/1/module-builds/?submitted_after=2016-09-03T12:25:00Z')
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], 24)
self.assertEquals(data['meta']['total'], 23)
def test_query_builds_filter_modified_before(self):
rv = self.client.get(
@@ -174,7 +175,7 @@ class TestViews(unittest.TestCase):
rv = self.client.get(
'/module-build-service/1/module-builds/?modified_after=2016-09-03T12:25:00Z')
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], 25)
self.assertEquals(data['meta']['total'], 24)
def test_query_builds_filter_owner(self):
rv = self.client.get(
@@ -186,7 +187,7 @@ class TestViews(unittest.TestCase):
rv = self.client.get(
'/module-build-service/1/module-builds/?state=3')
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], 21)
self.assertEquals(data['meta']['total'], 20)
def test_query_builds_two_filters(self):
rv = self.client.get('/module-build-service/1/module-builds/?owner=Moe%20Szyslak'
@@ -212,14 +213,14 @@ class TestViews(unittest.TestCase):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertEquals(data['component_builds'], [61])
self.assertEquals(data['name'], 'fakemodule')
self.assertEquals(data['scmurl'],
('git://pkgs.stg.fedoraproject.org/modules/testmodule'
'.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'))
'.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'))
self.assertEquals(data['version'], '5')
self.assertTrue(data['time_submitted'] is not None)
self.assertTrue(data['time_modified'] is not None)
@@ -227,9 +228,9 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['time_completed'], None)
self.assertEquals(data['stream'], '4.3.44')
self.assertEquals(data['owner'], 'Homer J. Simpson')
self.assertEquals(data['id'], 32)
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'wait')
self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/32')
self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31')
mmd = _modulemd.ModuleMetadata()
mmd.loads(data["modulemd"])
@@ -242,14 +243,14 @@ class TestViews(unittest.TestCase):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertEquals(data['component_builds'], [])
self.assertEquals(data['name'], 'fakemodule2')
self.assertEquals(data['scmurl'],
('git://pkgs.stg.fedoraproject.org/modules/testmodule'
'.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'))
'.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'))
self.assertEquals(data['version'], '5')
self.assertTrue(data['time_submitted'] is not None)
self.assertTrue(data['time_modified'] is not None)
@@ -257,13 +258,13 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['time_completed'], None)
self.assertEquals(data['stream'], '4.3.44')
self.assertEquals(data['owner'], 'Homer J. Simpson')
self.assertEquals(data['id'], 32)
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'wait')
def test_submit_build_auth_error(self):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#48932b90de214d9d13feefbd35246a81b6cb8d49'}))
'testmodule.git?#48931b90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertEquals(
data['message'],
@@ -309,7 +310,7 @@ class TestViews(unittest.TestCase):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertTrue(data['message'].startswith('Invalid modulemd:'))
self.assertEquals(data['status'], 422)
@@ -330,19 +331,19 @@ class TestViews(unittest.TestCase):
start = time.time()
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertEquals(len(data['component_builds']), 5)
self.assertEquals(data['name'], 'base-runtime')
self.assertEquals(data['scmurl'],
('git://pkgs.stg.fedoraproject.org/modules/testmodule'
'.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'))
'.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'))
self.assertTrue(data['time_submitted'] is not None)
self.assertTrue(data['time_modified'] is not None)
self.assertEquals(data['time_completed'], None)
self.assertEquals(data['owner'], 'Homer J. Simpson')
self.assertEquals(data['id'], 32)
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'wait')
# SCM availability check is parallelized, so 5 components should not
@@ -364,7 +365,7 @@ class TestViews(unittest.TestCase):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertEquals(data['status'], 422)
@@ -381,14 +382,14 @@ class TestViews(unittest.TestCase):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
{'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
'testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'}))
data = json.loads(rv.data)
self.assertEquals(data['component_builds'], [61, 62])
self.assertEquals(data['name'], 'fakemodule')
self.assertEquals(data['scmurl'],
('git://pkgs.stg.fedoraproject.org/modules/testmodule'
'.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'))
'.git?#68931c90de214d9d13feefbd35246a81b6cb8d49'))
self.assertEquals(data['version'], '5')
self.assertTrue(data['time_submitted'] is not None)
self.assertTrue(data['time_modified'] is not None)
@@ -396,12 +397,12 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['time_completed'], None)
self.assertEquals(data['stream'], '4.3.44')
self.assertEquals(data['owner'], 'Homer J. Simpson')
self.assertEquals(data['id'], 32)
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'wait')
self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/32')
self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31')
batches = {}
for build in ComponentBuild.query.filter_by(module_id=32).all():
for build in ComponentBuild.query.filter_by(module_id=31).all():
batches[build.package] = build.batch
self.assertEquals(batches["bash"], 2)
@@ -411,7 +412,7 @@ class TestViews(unittest.TestCase):
@patch('module_build_service.auth.assert_is_packager')
def test_cancel_build(self, mocked_assert_is_packager,
mocked_get_username):
rv = self.client.patch('/module-build-service/1/module-builds/31',
rv = self.client.patch('/module-build-service/1/module-builds/30',
data=json.dumps({'state': 'failed'}))
data = json.loads(rv.data)
@@ -434,7 +435,7 @@ class TestViews(unittest.TestCase):
@patch('module_build_service.auth.assert_is_packager')
def test_cancel_build_wrong_param(self, mocked_assert_is_packager,
mocked_get_username):
rv = self.client.patch('/module-build-service/1/module-builds/31',
rv = self.client.patch('/module-build-service/1/module-builds/30',
data=json.dumps({'some_param': 'value'}))
data = json.loads(rv.data)
@@ -447,7 +448,7 @@ class TestViews(unittest.TestCase):
@patch('module_build_service.auth.assert_is_packager')
def test_cancel_build_wrong_state(self, mocked_assert_is_packager,
mocked_get_username):
rv = self.client.patch('/module-build-service/1/module-builds/31',
rv = self.client.patch('/module-build-service/1/module-builds/30',
data=json.dumps({'state': 'some_state'}))
data = json.loads(rv.data)