modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 03:38:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'api-search'

Browse Source
This commit is contained in:
Nils Philippsen
2016-08-24 15:23:15 +02:00
parent 66764e2d5b 015990bc6c
commit 343ad70c5a
8 changed files with 282 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
README.rst
View File

@@ -84,10 +84,14 @@ their states.
Listing all module builds
-------------------------
The list of all tracked builds and their states can be obtained by querying the
"module-builds" resource.
The list of all tracked builds and their states can be obtained by querying the "module-builds" resource.
There are a number of configurable GET parameters to change how the module builds are displayed. These parameters are:
::
- verbose - Shows the builds with the same amount of detail as querying them individually (i.e. verbose=True). This value defaults to False.
- page - Specifies which page should be displayed (e.g. page=3). This value defaults to 1.
- per_page - Specifies how many items per page should be displayed (e.g. per_page=20). This value defaults to 10.
An example of querying the "module-builds" resource without any additional parameters::
GET /rida/module-builds/
@@ -152,11 +156,7 @@ The list of all tracked builds and their states can be obtained by querying the
}
The API is paginated, and defaults to 10 items per page. These values are configurable with the `page` and `per_page`
GET parameters respectively. Additionally, there is a `verbose` parameter that defaults to false, which allows you to
query all the builds with the same amount of detail as querying them individually.
::
An example of querying the "module-builds" resource with the "verbose", "per_page", and the "page" parameters::
GET /rida/module-builds/?verbose=true&per_page=3&page=1
@@ -170,27 +170,42 @@ query all the builds with the same amount of detail as querying them individuall
"items": [
{
"id": 1,
"name": "testmodule",
"owner": "mprahl",
"state": 3,
"tasks": {
"rpms/bash": "90109464/1",
"rpms/module-build-macros": "90109446/1"
}
},
"time_completed": "2016-08-22T09:44:11Z",
"time_modified": "2016-08-22T09:44:11Z",
"time_submitted": "2016-08-22T09:40:07Z"
},
{
"id": 2,
"name": "testmodule",
"owner": "ralph",
"state": 3,
"tasks": {
"rpms/bash": "90109465/1",
"rpms/module-build-macros": "90109450/1"
}
},
"time_completed": "2016-08-22T09:54:04Z",
"time_modified": "2016-08-22T09:54:04Z",
"time_submitted": "2016-08-22T09:48:11Z"
},
{
"id": 3,
"name": "testmodule",
"owner": "mprahl",
"state": 3,
"tasks": {
"rpms/bash": "90109497/1",
"rpms/module-build-macros": "90109480/1"
}
},
"time_completed": "2016-08-22T10:05:08Z",
"time_modified": "2016-08-22T10:05:08Z",
"time_submitted": "2016-08-22T09:58:04Z"
}
],
"meta": {
@@ -205,6 +220,54 @@ query all the builds with the same amount of detail as querying them individuall
}
Filtering module builds
-----------------------
The module-builds can be filtered by a variety of GET parameters. These paramters are:
- owner - Shows builds submitted by a particular user (e.g. owner=mprahl)
- state - Shows builds in a particular state (can be the state name or the state ID) (e.g. state=done)
- submitted_before - Shows builds that were submitted before a particular Zulu ISO 8601 timestamp (e.g. submitted_before=2016-08-23T09:40:07Z)
- submitted_after - Shows builds that were submitted after a particular Zulu ISO 8601 timestamp (e.g. submitted_after=2016-08-22T09:40:07Z)
- modified_before - Shows builds that were modified before a particular Zulu ISO 8601 timestamp (e.g. modified_before=2016-08-23T09:40:07Z)
- modified_after - Shows builds that were modified after a particular Zulu ISO 8601 timestamp (e.g. modified_after=2016-08-22T09:40:07Z)
- completed_before - Shows builds that were completed before a particular Zulu ISO 8601 timestamp (e.g. completed_before=2016-08-22T09:40:07Z)
- completed_after - Shows builds that were completed after a particular Zulu ISO 8601 timestamp (e.g. completed_after=2016-08-23T09:40:07Z)
An example of querying the "module-builds" resource with the "state", and the "submitted_before" parameters::
GET /rida/module-builds/?state=done&submitted_before=2016-08-23T08:10:07Z
::
HTTP 200 OK
::
{
"items": [
{
"id": 1,
"state": 3
},
{
"id": 2,
"state": 3
},
{
"id": 3,
"state": 3
}
],
"meta": {
"first": "https://127.0.0.1:5000/rida/module-builds/?per_page=10&page=1",
"last": "https://127.0.0.1:5000/rida/module-builds/?per_page=10&page=1",
"page": 1,
"pages": 1,
"per_page": 3,
"total": 3
}
HTTP Response Codes
-------------------

2
Vagrantfile vendored
View File

@@ -13,7 +13,7 @@ $script = <<SCRIPT
SCRIPT
Vagrant.configure("2") do |config|
config.vm.box = "box-cutter/fedora23"
config.vm.box = "boxcutter/fedora24"
config.vm.synced_folder "./", "/opt/fm-orchestrator/src"
config.vm.network "forwarded_port", guest: 5000, host: 5000
config.vm.provision "shell", inline: $script

41
migrations/versions/1a44272e8b4c_.py Normal file
View File

@@ -0,0 +1,41 @@
"""Adds the owner, time_completed, time_modified, and time_submitted columns to the module_builds table
Revision ID: 1a44272e8b4c
Revises: a7a553e5ca1d
Create Date: 2016-08-17 17:00:31.126429
"""
# revision identifiers, used by Alembic.
revision = '1a44272e8b4c'
down_revision = 'a7a553e5ca1d'
from alembic import op
import sqlalchemy as sa
from datetime import datetime
epoch = datetime.utcfromtimestamp(0).strftime('%Y-%m-%d %H:%M:%S')
def upgrade():
op.add_column('module_builds', sa.Column('owner', sa.String(), server_default='Unknown User', nullable=False))
op.add_column('module_builds', sa.Column('time_completed', sa.DateTime(), nullable=True, server_default=epoch))
op.add_column('module_builds', sa.Column('time_modified', sa.DateTime(), nullable=True, server_default=epoch))
op.add_column('module_builds', sa.Column('time_submitted', sa.DateTime(), nullable=False, server_default=epoch))
# Remove migration-only defaults. Using batch_alter_table() recreates the table instead of using ALTER COLUMN
# on simplistic DB engines. Thanks SQLite!
with op.batch_alter_table('module_builds') as b:
b.alter_column('owner', server_default=None)
b.alter_column('time_completed', server_default=None)
b.alter_column('time_modified', server_default=None)
b.alter_column('time_submitted', server_default=None)
def downgrade():
# Thanks again!
with op.batch_alter_table('module_builds') as b:
b.drop_column('time_submitted')
b.drop_column('time_modified')
b.drop_column('time_completed')
b.drop_column('owner')

6
migrations/versions/a7a553e5ca1d_.py
View File

@@ -1,4 +1,4 @@
"""empty message
"""Initial database migration script that creates the database tables
Revision ID: a7a553e5ca1d
Revises: None
@@ -15,7 +15,6 @@ import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('modules',
sa.Column('name', sa.String(), nullable=False),
sa.PrimaryKeyConstraint('name')
@@ -46,12 +45,9 @@ def upgrade():
sa.ForeignKeyConstraint(['module_id'], ['module_builds.id'], ),
sa.PrimaryKeyConstraint('id')
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('component_builds')
op.drop_table('module_builds')
op.drop_table('modules')
### end Alembic commands ###

25
rida/errors.py Normal file
View File

@@ -0,0 +1,25 @@
# Copyright (c) 2016 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Matt Prahl <mprahl@redhat.com>
""" Defines custom exceptions and error handling functions """
class ValidationError(ValueError):
pass

50
rida/models.py
View File

@@ -25,12 +25,11 @@
""" SQLAlchemy Database models for the Flask app
"""
from rida import db, log
from datetime import datetime
from sqlalchemy.orm import validates
import modulemd as _modulemd
from rida import db, log
import rida.messaging
@@ -83,6 +82,10 @@ class ModuleBuild(RidaBase):
modulemd = db.Column(db.String, nullable=False)
koji_tag = db.Column(db.String) # This gets set after 'wait'
scmurl = db.Column(db.String)
owner = db.Column(db.String, nullable=False)
time_submitted = db.Column(db.DateTime, nullable=False)
time_modified = db.Column(db.DateTime)
time_completed = db.Column(db.DateTime)
# A monotonically increasing integer that represents which batch or
# iteration this module is currently on for successive rebuilds of its
@@ -125,7 +128,8 @@ class ModuleBuild(RidaBase):
return session.query(cls).filter(cls.id==event['msg']['id']).first()
@classmethod
def create(cls, session, conf, name, version, release, modulemd, scmurl):
def create(cls, session, conf, name, version, release, modulemd, scmurl, username):
now = datetime.utcnow()
module = cls(
name=name,
version=version,
@@ -133,6 +137,8 @@ class ModuleBuild(RidaBase):
state="init",
modulemd=modulemd,
scmurl=scmurl,
owner=username,
time_submitted=now
)
session.add(module)
session.commit()
@@ -146,8 +152,14 @@ class ModuleBuild(RidaBase):
def transition(self, conf, state):
""" Record that a build has transitioned state. """
now = datetime.utcnow()
old_state = self.state
self.state = state
self.time_modified = now
if self.state in ['done', 'failed']:
self.time_completed = now
log.debug("%r, state %r->%r" % (self, old_state, self.state))
rida.messaging.publish(
modname='rida',
@@ -187,11 +199,41 @@ class ModuleBuild(RidaBase):
'state': self.state,
'state_name': INVERSE_BUILD_STATES[self.state],
'scmurl': self.scmurl,
'owner': self.owner,
'time_submitted': self.time_submitted,
'time_modified': self.time_modified,
'time_completed': self.time_completed,
# TODO, show their entire .json() ?
'component_builds': [build.id for build in self.component_builds],
}
@staticmethod
def _utc_datetime_to_iso(datetime_object):
"""
Takes a UTC datetime object and returns an ISO formatted string
:param datetime_object: datetime.datetime
:return: string with datetime in ISO format
"""
if datetime_object:
# Converts the datetime to ISO 8601
return datetime_object.strftime("%Y-%m-%dT%H:%M:%SZ")
return None
def api_json(self):
return {
"id": self.id,
"state": self.state,
"owner": self.owner,
"name": self.name,
"time_submitted": self._utc_datetime_to_iso(self.time_submitted),
"time_modified": self._utc_datetime_to_iso(self.time_modified),
"time_completed": self._utc_datetime_to_iso(self.time_completed),
"tasks": self.tasks()
}
def tasks(self):
"""
:return: dictionary containing the tasks associated with the build

61
rida/utils.py
View File

@@ -22,9 +22,12 @@
# Matt Prahl <mprahl@redhat.com>
""" Utility functions for rida. """
from flask import request, url_for
from datetime import datetime
import re
import functools
import time
from rida import log, models
from errors import ValidationError
def retry(timeout=120, interval=30, wait_on=Exception):
@@ -96,3 +99,61 @@ def pagination_metadata(p_query):
per_page=p_query.per_page, _external=True)
return pagination_data
def filter_module_builds(flask_request):
"""
Returns a flask_sqlalchemy.Pagination object based on the request parameters
:param request: Flask request object
:return: flask_sqlalchemy.Pagination
"""
search_query = dict()
state = flask_request.args.get('state', None)
if state:
if state.isdigit():
search_query['state'] = state
else:
if state in models.BUILD_STATES:
search_query['state'] = models.BUILD_STATES[state]
else:
raise ValidationError('An invalid state was supplied')
for key in ['name', 'owner']:
if flask_request.args.get(key, None):
search_query[key] = flask_request.args[key]
query = models.ModuleBuild.query
if search_query:
query = query.filter_by(**search_query)
# This is used when filtering the date request parameters, but it is here to avoid recompiling
utc_iso_datetime_regex = re.compile(r'^(?P<datetime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?'
r'(?:Z|[-+]00(?::00)?)?$')
# Filter the query based on date request parameters
for item in ('submitted', 'modified', 'completed'):
for context in ('before', 'after'):
request_arg = '%s_%s' % (item, context) # i.e. submitted_before
iso_datetime_arg = request.args.get(request_arg, None)
if iso_datetime_arg:
iso_datetime_matches = re.match(utc_iso_datetime_regex, iso_datetime_arg)
if not iso_datetime_matches or not iso_datetime_matches.group('datetime'):
raise ValidationError('An invalid Zulu ISO 8601 timestamp was provided for the "%s" parameter'
% request_arg)
# Converts the ISO 8601 string to a datetime object for SQLAlchemy to use to filter
item_datetime = datetime.strptime(iso_datetime_matches.group('datetime'), '%Y-%m-%dT%H:%M:%S')
# Get the database column to filter against
column = getattr(models.ModuleBuild, 'time_' + item)
if context == 'after':
query = query.filter(column >= item_datetime)
elif context == 'before':
query = query.filter(column <= item_datetime)
page = flask_request.args.get('page', 1, type=int)
per_page = flask_request.args.get('per_page', 10, type=int)
return query.paginate(page, per_page, False)

50
rida/views.py
View File

@@ -39,38 +39,45 @@ import shutil
import tempfile
from rida import app, conf, db, log
from rida import models
from rida.utils import pagination_metadata
from rida.utils import pagination_metadata, filter_module_builds
from errors import ValidationError
@app.route("/rida/module-builds/", methods=["POST"])
def submit_build():
"""Handles new module build submissions."""
username = rida.auth.is_packager(conf.pkgdb_api_url)
if not username:
return ("You must use your Fedora certificate when submitting"
" new build", 403)
return "You must use your Fedora certificate when submitting a new build", 403
try:
r = json.loads(request.get_data().decode("utf-8"))
except:
return "Invalid JSON submitted", 400
if "scmurl" not in r:
return "Missing scmurl", 400
url = r["scmurl"]
urlallowed = False
for prefix in conf.scmurls:
if url.startswith(prefix):
urlallowed = True
break
if not urlallowed:
return "The submitted scmurl isn't allowed", 403
yaml = str()
td = None
try:
td = tempfile.mkdtemp()
scm = rida.scm.SCM(url, conf.scmurls)
cod = scm.checkout(td)
cofn = os.path.join(cod, (scm.name + ".yaml"))
with open(cofn, "r") as mmdfile:
yaml = mmdfile.read()
except Exception as e:
@@ -82,12 +89,20 @@ def submit_build():
rc = 500
return str(e), rc
finally:
shutil.rmtree(td)
try:
if td is not None:
shutil.rmtree(td)
except Exception as e:
log.warning(
"Failed to remove temporary directory {!r}: {}".format(
td, str(e)))
mmd = modulemd.ModuleMetadata()
try:
mmd.loads(yaml)
except:
return "Invalid modulemd", 422
if models.ModuleBuild.query.filter_by(name=mmd.name, version=mmd.version, release=mmd.release).first():
return "Module already exists", 409
@@ -99,6 +114,7 @@ def submit_build():
release=mmd.release,
modulemd=yaml,
scmurl=url,
username=username
)
def failure(message, code):
@@ -123,9 +139,12 @@ def submit_build():
pkg["commit"] = rida.scm.SCM(pkg["repository"]).get_latest()
except Exception as e:
return failure("Failed to get the latest commit: %s" % pkgname, 422)
full_url = pkg["repository"] + "?#" + pkg["commit"]
if not rida.scm.SCM(full_url).is_available():
return failure("Cannot checkout %s" % pkgname, 422)
build = models.ComponentBuild(
module_id=module.id,
package=pkgname,
@@ -133,6 +152,7 @@ def submit_build():
scmurl=full_url,
)
db.session.add(build)
module.modulemd = mmd.dumps()
module.transition(conf, models.BUILD_STATES["wait"])
db.session.add(module)
@@ -145,18 +165,19 @@ def submit_build():
@app.route("/rida/module-builds/", methods=["GET"])
def query_builds():
"""Lists all tracked module builds."""
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 10, type=int)
p_query = models.ModuleBuild.query.paginate(page, per_page, False)
verbose_flag = request.args.get('verbose', 'false')
try:
p_query = filter_module_builds(request)
except ValidationError as e:
return e.message, 400
json_data = {
'meta': pagination_metadata(p_query)
}
verbose_flag = request.args.get('verbose', 'false')
if verbose_flag.lower() == 'true' or verbose_flag == '1':
json_data['items'] = [{'id': item.id, 'state': item.state, 'tasks': item.tasks()}
for item in p_query.items]
json_data['items'] = [item.api_json() for item in p_query.items]
else:
json_data['items'] = [{'id': item.id, 'state': item.state} for item in p_query.items]
@@ -169,11 +190,6 @@ def query_build(id):
module = models.ModuleBuild.query.filter_by(id=id).first()
if module:
return jsonify({
"id": module.id,
"state": module.state,
"tasks": module.tasks()
}), 200
return jsonify(module.api_json()), 200
else:
return "No such module found.", 404