modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-02 20:59:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Changes in MBS restful API:

Browse Source 
- Refactor MBS API code
- Unify module-/component_build API philosophy/design/approach
- Naming fixes
- _utc_datetime_to_iso moved from ModuleBuildAPI and is now a module-level function.
- Existing v1 API remains unchanged. ComponentBuildAPI now supports individual component build listing + verbose mode.
- documented in README
- various component_build API tests added
This commit is contained in:
Filip Valder
2017-09-04 15:04:11 +02:00
parent 6cb836c862
commit 3cb41aa5dc
8 changed files with 175 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
README.rst
View File

@@ -79,10 +79,9 @@ builds. For more info, there's an existing help available.
Client-side API
===============
The orchestrator implements a RESTful interface for module build
submission and state querying. Not all REST methods are supported. See
below for details. For client tooling which utilizes the API, please
refer to `Client tooling`_ section.
The MBS implements a RESTful interface for module build submission and state
querying. Not all REST methods are supported. See below for details. For client
tooling which utilizes the API, please refer to `Client tooling`_ section.
Module build submission
-----------------------
@@ -158,11 +157,11 @@ about the referenced build task.
...
}
"id" is the ID of the task. "state" refers to the orchestrator module
build state and might be one of "init", "wait", "build", "done", "failed" or
"ready". "tasks" is a dictionary of information about the individual component
builds including their IDs in the backend buildsystem, their state, a reason
for their state, and the NVR (if known).
"id" is the ID of the task. "state" refers to the MBS module build state and
might be one of "init", "wait", "build", "done", "failed" or "ready". "tasks"
is a dictionary of information about the individual component builds including
their IDs in the backend buildsystem, their state, a reason for their state,
and the NVR (if known).
By adding ``?verbose=1`` to the request, additional detailed information
about the module can be obtained.
@@ -409,6 +408,49 @@ and the "submitted_before" parameters::
"total": 3
}
Component build state query
---------------------------
Getting particular component build is very similar to a module build query.
::
GET /module-build-service/1/component-builds/1
The response, if the build exists, would include various pieces of information
about the referenced component build.
::
HTTP 200 OK
::
{
"format": "rpms",
"id": 1,
"module_build": 1,
"package": "nginx",
"state": 1,
"state_name": "COMPLETE",
"state_reason": null,
"task_id": 12312345
}
"id" is the ID of the component build. "state_name" refers to the MBS component
build state and might be one of "COMPLETE", "FAILED", "CANCELED". "task_id"
is a related task ID in the backend buildsystem, their state and a reason
for their state. "module_build" refers to the module build ID for which this
component was built. "format" is typically "rpms", since we're building it
and "package" is simply the package name.
By adding ``?verbose=1`` to the request, additional detailed information
about the component can be obtained.
::
GET /module-build-service/1/component-builds/1?verbose=1
Listing component builds
------------------------

96
module_build_service/models.py
View File

@@ -79,6 +79,19 @@ BUILD_STATES = {
INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()}
def _utc_datetime_to_iso(datetime_object):
"""
Takes a UTC datetime object and returns an ISO formatted string
:param datetime_object: datetime.datetime
:return: string with datetime in ISO format
"""
if datetime_object:
# Converts the datetime to ISO 8601
return datetime_object.strftime("%Y-%m-%dT%H:%M:%SZ")
return None
@contextlib.contextmanager
def _dummy_context_mgr():
"""
@@ -290,8 +303,8 @@ class ModuleBuild(MBSBase):
return []
local_modules = [m for m in local_modules
if m.koji_tag
and m.koji_tag.startswith(conf.mock_resultsdir)]
if m.koji_tag and
m.koji_tag.startswith(conf.mock_resultsdir)]
return local_modules
@classmethod
@@ -335,59 +348,37 @@ class ModuleBuild(MBSBase):
def json(self):
return {
'id': self.id,
'name': self.name,
'stream': self.stream,
'version': self.version,
'state': self.state,
'state_name': INVERSE_BUILD_STATES[self.state],
'state_reason': self.state_reason,
'state_url': get_url_for('module_build', id=self.id),
'scmurl': self.scmurl,
'owner': self.owner,
'time_submitted': self._utc_datetime_to_iso(self.time_submitted),
'time_modified': self._utc_datetime_to_iso(self.time_modified),
'time_completed': self._utc_datetime_to_iso(self.time_completed),
"tasks": self.tasks(),
'name': self.name,
'scmurl': self.scmurl,
'time_submitted': _utc_datetime_to_iso(self.time_submitted),
'time_modified': _utc_datetime_to_iso(self.time_modified),
'time_completed': _utc_datetime_to_iso(self.time_completed),
'koji_tag': self.koji_tag,
'tasks': self.tasks(),
}
def extended_json(self):
json = self.json()
json.update({
'stream': self.stream,
'version': self.version,
'state_url': get_url_for('module_build', id=self.id),
# TODO, show their entire .json() ?
'component_builds': [build.id for build in self.component_builds],
'modulemd': self.modulemd,
'koji_tag': self.koji_tag,
'state_trace': [{'time': self._utc_datetime_to_iso(record.state_time),
'state_trace': [{'time': _utc_datetime_to_iso(record.state_time),
'state': record.state,
'state_name': INVERSE_BUILD_STATES[record.state],
'reason': record.state_reason}
for record
in self.state_trace(self.id)]
}
})
@staticmethod
def _utc_datetime_to_iso(datetime_object):
"""
Takes a UTC datetime object and returns an ISO formatted string
:param datetime_object: datetime.datetime
:return: string with datetime in ISO format
"""
if datetime_object:
# Converts the datetime to ISO 8601
return datetime_object.strftime("%Y-%m-%dT%H:%M:%SZ")
return None
def api_json(self):
return {
"id": self.id,
"state": self.state,
'state_name': INVERSE_BUILD_STATES[self.state],
'state_reason': self.state_reason,
"owner": self.owner,
"name": self.name,
"scmurl": self.scmurl,
"time_submitted": self._utc_datetime_to_iso(self.time_submitted),
"time_modified": self._utc_datetime_to_iso(self.time_modified),
"time_completed": self._utc_datetime_to_iso(self.time_completed),
"koji_tag": self.koji_tag,
"tasks": self.tasks()
}
return json
def tasks(self):
"""
@@ -435,7 +426,7 @@ class ModuleBuildTrace(MBSBase):
retval = {
'id': self.id,
'module_id': self.module_id,
'state_time': self._utc_datetime_to_iso(self.state_time),
'state_time': _utc_datetime_to_iso(self.state_time),
'state': self.state,
'state_reason': self.state_reason,
}
@@ -495,6 +486,10 @@ class ComponentBuild(MBSBase):
return session.query(cls).filter_by(
package=component_name, module_id=module_id).first()
def state_trace(self, component_id):
return ComponentBuildTrace.query.filter_by(
component_id=component_id).order_by(ComponentBuildTrace.state_time).all()
def json(self):
retval = {
'id': self.id,
@@ -516,6 +511,19 @@ class ComponentBuild(MBSBase):
return retval
def extended_json(self):
json = self.json()
json.update({
'state_trace': [{'time': _utc_datetime_to_iso(record.state_time),
'state': record.state,
'state_name': INVERSE_BUILD_STATES[record.state],
'reason': record.state_reason}
for record
in self.state_trace(self.id)]
})
return json
def __repr__(self):
return "<ComponentBuild %s, %r, state: %r, task_id: %r, batch: %r, state_reason: %s>" % (
self.package, self.module_id, self.state, self.task_id, self.batch, self.state_reason)
@@ -536,7 +544,7 @@ class ComponentBuildTrace(MBSBase):
retval = {
'id': self.id,
'component_id': self.component_id,
'state_time': self._utc_datetime_to_iso(self.state_time),
'state_time': _utc_datetime_to_iso(self.state_time),
'state': self.state,
'state_reason': self.state_reason,
'task_id': self.task_id,

2
module_build_service/scheduler/handlers/modules.py
View File

@@ -150,7 +150,7 @@ def wait(config, session, msg):
log.info("Found build=%r from message" % build)
log.info("%r", build.modulemd)
module_info = build.json()
module_info = build.extended_json()
if module_info['state'] != msg.module_build_state:
log.warn("Note that retrieved module state %r "
"doesn't match message module state %r" % (

42
module_build_service/views.py
View File

@@ -67,6 +67,12 @@ api_v1 = {
'methods': ['GET'],
}
},
'component_build': {
'url': '/module-build-service/1/component-builds/<int:id>',
'options': {
'methods': ['GET'],
}
},
}
@@ -84,23 +90,23 @@ class ComponentBuildAPI(MethodView):
}
if verbose_flag.lower() == 'true' or verbose_flag == '1':
json_data['items'] = [item.api_json() for item in p_query.items]
json_data['items'] = [item.extended_json() for item in p_query.items]
else:
json_data['items'] = [{'id': item.id, 'state': item.state} for
item in p_query.items]
return jsonify(json_data), 200
else:
# Lists details for the specified module builds
module = models.ComponentBuild.query.filter_by(id=id).first()
# Lists details for the specified component builds
component = models.ComponentBuild.query.filter_by(id=id).first()
if module:
if component:
if verbose_flag.lower() == 'true' or verbose_flag == '1':
return jsonify(module.json()), 200
return jsonify(component.extended_json()), 200
else:
return jsonify(module.api_json()), 200
return jsonify(component.json()), 200
else:
raise NotFound('No such module found.')
raise NotFound('No such component found.')
class ModuleBuildAPI(MethodView):
@@ -117,7 +123,7 @@ class ModuleBuildAPI(MethodView):
}
if verbose_flag.lower() == 'true' or verbose_flag == '1':
json_data['items'] = [item.api_json() for item in p_query.items]
json_data['items'] = [item.extended_json() for item in p_query.items]
else:
json_data['items'] = [{'id': item.id, 'state': item.state} for
item in p_query.items]
@@ -129,9 +135,9 @@ class ModuleBuildAPI(MethodView):
if module:
if verbose_flag.lower() == 'true' or verbose_flag == '1':
return jsonify(module.json()), 200
return jsonify(module.extended_json()), 200
else:
return jsonify(module.api_json()), 200
return jsonify(module.json()), 200
else:
raise NotFound('No such module found.')
@@ -150,7 +156,7 @@ class ModuleBuildAPI(MethodView):
handler.validate()
module = handler.post()
return jsonify(module.json()), 201
return jsonify(module.extended_json()), 201
def patch(self, id):
username, groups = module_build_service.auth.get_user(request)
@@ -194,7 +200,7 @@ class ModuleBuildAPI(MethodView):
db.session.add(module)
db.session.commit()
return jsonify(module.api_json()), 200
return jsonify(module.json()), 200
class BaseHandler(object):
@@ -286,15 +292,17 @@ def register_api_v1():
module_view = ModuleBuildAPI.as_view('module_builds')
component_view = ComponentBuildAPI.as_view('component_builds')
for key, val in api_v1.items():
if key != 'component_builds_list':
if key.startswith('component_build'):
app.add_url_rule(val['url'],
endpoint=key,
view_func=component_view,
**val['options'])
elif key.startswith('module_build'):
app.add_url_rule(val['url'],
endpoint=key,
view_func=module_view,
**val['options'])
else:
app.add_url_rule(val['url'],
endpoint=key,
view_func=component_view,
**val['options'])
raise NotImplementedError("Unhandled api key.")
register_api_v1()

1
test-requirements.txt
View File

@@ -1,5 +1,6 @@
copr
mock
nose
parameterized
pytest
vcrpy

2
tests/test_scheduler/test_module_wait.py
View File

@@ -66,7 +66,7 @@ class TestModuleWait(unittest.TestCase):
builder.module_build_tag = {'name': 'some-tag-build'}
create_builder.return_value = builder
mocked_module_build = mock.Mock()
mocked_module_build.json.return_value = {
mocked_module_build.extended_json.return_value = {
'name': 'foo',
'stream': 1,
'version': 1,

45
tests/test_views/test_views.py
View File

@@ -32,6 +32,7 @@ from mock import patch, PropertyMock, MagicMock
from shutil import copyfile
from os import path, mkdir
from os.path import dirname
from parameterized import parameterized
import hashlib
from tests import app, init_data
@@ -256,12 +257,52 @@ class TestViews(unittest.TestCase):
self.assertEquals(item['time_modified'], '2016-09-03T11:25:32Z')
self.assertEquals(item['time_submitted'], '2016-09-03T11:23:20Z')
def test_query_builds_filter_nvr(self):
def test_query_component_build(self):
rv = self.client.get('/module-build-service/1/component-builds/1')
data = json.loads(rv.data)
self.assertEquals(data['id'], 1)
self.assertEquals(data['format'], 'rpms')
self.assertEquals(data['module_build'], 1)
self.assertEquals(data['package'], 'nginx')
self.assertEquals(data['state'], 1)
self.assertEquals(data['state_name'], 'COMPLETE')
self.assertEquals(data['state_reason'], None)
self.assertEquals(data['task_id'], 12312345)
def test_query_component_build_verbose(self):
rv = self.client.get('/module-build-service/1/component-builds/3?verbose=1')
data = json.loads(rv.data)
self.assertEquals(data['id'], 3)
self.assertEquals(data['format'], 'rpms')
self.assertEquals(data['module_build'], 2)
self.assertEquals(data['package'], 'postgresql')
self.assertEquals(data['state'], 1)
self.assertEquals(data['state_name'], 'COMPLETE')
self.assertEquals(data['state_reason'], None)
self.assertEquals(data['task_id'], 2433433)
self.assertEquals(data['state_trace'][0]['reason'], None)
self.assertTrue(data['state_trace'][0]['time'] is not None)
self.assertEquals(data['state_trace'][0]['state'], 1)
self.assertEquals(data['state_trace'][0]['state_name'], 'wait')
component_builds_filters = ['tagged', 'ref', 'format']
@parameterized.expand([
('format', 'rpms', 60),
('ref', 'this-filter-query-should-return-zero-items', 0),
('tagged', 'this-filter-query-should-return-zero-items', 0),
])
def test_query_component_builds_filters(self, f, s, c):
rv = self.client.get('/module-build-service/1/component-builds/?{}={}'.format(f, s))
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], c)
def test_query_component_builds_filter_nvr(self):
rv = self.client.get('/module-build-service/1/component-builds/?nvr=nginx-1.10.1-2.module_nginx_1_2')
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], 10)
def test_query_builds_filter_task_id(self):
def test_query_component_builds_filter_task_id(self):
rv = self.client.get('/module-build-service/1/component-builds/?task_id=12312346')
data = json.loads(rv.data)
self.assertEquals(data['meta']['total'], 1)

1
tox.ini
View File

@@ -14,6 +14,7 @@ deps =
copr
mock
nose
parameterized
pytest
vcrpy
commands = py.test {posargs}