From 1386d3506ce87a542fec353b07b807e25874791f Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Wed, 17 Aug 2016 10:19:57 -0400 Subject: [PATCH 01/10] Modify formatting for readability Also, remove misleading comment. --- rida/views.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/rida/views.py b/rida/views.py index 4975c5a2..f629def6 100644 --- a/rida/views.py +++ b/rida/views.py @@ -45,32 +45,38 @@ from rida.utils import pagination_metadata @app.route("/rida/module-builds/", methods=["POST"]) def submit_build(): """Handles new module build submissions.""" - + # Get the time from when the build was submitted username = rida.auth.is_packager(conf.pkgdb_api_url) if not username: - return ("You must use your Fedora certificate when submitting" - " new build", 403) + return "You must use your Fedora certificate when submitting a new build", 403 try: r = json.loads(request.get_data().decode("utf-8")) except: return "Invalid JSON submitted", 400 + if "scmurl" not in r: return "Missing scmurl", 400 + url = r["scmurl"] urlallowed = False + for prefix in conf.scmurls: + if url.startswith(prefix): urlallowed = True break + if not urlallowed: return "The submitted scmurl isn't allowed", 403 + yaml = str() try: td = tempfile.mkdtemp() scm = rida.scm.SCM(url, conf.scmurls) cod = scm.checkout(td) cofn = os.path.join(cod, (scm.name + ".yaml")) + with open(cofn, "r") as mmdfile: yaml = mmdfile.read() except Exception as e: @@ -83,11 +89,13 @@ def submit_build(): return str(e), rc finally: shutil.rmtree(td) + mmd = modulemd.ModuleMetadata() try: mmd.loads(yaml) except: return "Invalid modulemd", 422 + if models.ModuleBuild.query.filter_by(name=mmd.name, version=mmd.version, release=mmd.release).first(): return "Module already exists", 409 @@ -123,9 +131,12 @@ def submit_build(): pkg["commit"] = rida.scm.SCM(pkg["repository"]).get_latest() except Exception as e: return failure("Failed to get the latest commit: %s" % pkgname, 422) + full_url = pkg["repository"] + "?#" + pkg["commit"] + if not rida.scm.SCM(full_url).is_available(): return failure("Cannot checkout %s" % pkgname, 422) + build = models.ComponentBuild( module_id=module.id, package=pkgname, @@ -133,6 +144,7 @@ def submit_build(): scmurl=full_url, ) db.session.add(build) + module.modulemd = mmd.dumps() module.transition(conf, models.BUILD_STATES["wait"]) db.session.add(module) From d65f4fed55569c99c5e4d44e077ca2eae420b614 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Wed, 24 Aug 2016 14:53:01 +0200 Subject: [PATCH 02/10] Fix migration comments Fill out change log, remove automatically generated comments. --- migrations/versions/a7a553e5ca1d_.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/migrations/versions/a7a553e5ca1d_.py b/migrations/versions/a7a553e5ca1d_.py index 3723b98d..769c790e 100644 --- a/migrations/versions/a7a553e5ca1d_.py +++ b/migrations/versions/a7a553e5ca1d_.py @@ -1,4 +1,4 @@ -"""empty message +"""Initial database migration script that creates the database tables Revision ID: a7a553e5ca1d Revises: None @@ -15,7 +15,6 @@ import sqlalchemy as sa def upgrade(): - ### commands auto generated by Alembic - please adjust! ### op.create_table('modules', sa.Column('name', sa.String(), nullable=False), sa.PrimaryKeyConstraint('name') @@ -46,12 +45,9 @@ def upgrade(): sa.ForeignKeyConstraint(['module_id'], ['module_builds.id'], ), sa.PrimaryKeyConstraint('id') ) - ### end Alembic commands ### def downgrade(): - ### commands auto generated by Alembic - please adjust! ### op.drop_table('component_builds') op.drop_table('module_builds') op.drop_table('modules') - ### end Alembic commands ### From 1cd21434fb3e82ce6d1ddb290dac6ac39cd9a8fa Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Wed, 24 Aug 2016 14:58:45 +0200 Subject: [PATCH 03/10] catch exceptions when removing temporary directory Additionally, don't try to remove it when it wasn't created in the first place. --- rida/views.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rida/views.py b/rida/views.py index f629def6..cb18e69b 100644 --- a/rida/views.py +++ b/rida/views.py @@ -71,6 +71,7 @@ def submit_build(): return "The submitted scmurl isn't allowed", 403 yaml = str() + td = None try: td = tempfile.mkdtemp() scm = rida.scm.SCM(url, conf.scmurls) @@ -88,7 +89,13 @@ def submit_build(): rc = 500 return str(e), rc finally: - shutil.rmtree(td) + try: + if td is not None: + shutil.rmtree(td) + except Exception as e: + log.warning( + "Failed to remove temporary directory {!r}: {}".format( + td, str(e))) mmd = modulemd.ModuleMetadata() try: From d1a01e5d79faa67f03fc11539ccfa4e0ccda6fca Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Wed, 17 Aug 2016 10:27:16 -0400 Subject: [PATCH 04/10] Add owner and timestamp columns to the module_builds table --- migrations/versions/1a44272e8b4c_.py | 41 ++++++++++++++++++++++++++++ rida/models.py | 15 +++++++--- rida/views.py | 1 + 3 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 migrations/versions/1a44272e8b4c_.py diff --git a/migrations/versions/1a44272e8b4c_.py b/migrations/versions/1a44272e8b4c_.py new file mode 100644 index 00000000..463d87ca --- /dev/null +++ b/migrations/versions/1a44272e8b4c_.py @@ -0,0 +1,41 @@ +"""Adds the owner, time_completed, time_modified, and time_submitted columns to the module_builds table + +Revision ID: 1a44272e8b4c +Revises: a7a553e5ca1d +Create Date: 2016-08-17 17:00:31.126429 + +""" + +# revision identifiers, used by Alembic. +revision = '1a44272e8b4c' +down_revision = 'a7a553e5ca1d' + +from alembic import op +import sqlalchemy as sa +from datetime import datetime + +epoch = datetime.utcfromtimestamp(0).strftime('%Y-%m-%d %H:%M:%S') + + +def upgrade(): + op.add_column('module_builds', sa.Column('owner', sa.String(), server_default='Unknown User', nullable=False)) + op.add_column('module_builds', sa.Column('time_completed', sa.DateTime(), nullable=True, server_default=epoch)) + op.add_column('module_builds', sa.Column('time_modified', sa.DateTime(), nullable=True, server_default=epoch)) + op.add_column('module_builds', sa.Column('time_submitted', sa.DateTime(), nullable=False, server_default=epoch)) + + # Remove migration-only defaults. Using batch_alter_table() recreates the table instead of using ALTER COLUMN + # on simplistic DB engines. Thanks SQLite! + with op.batch_alter_table('module_builds') as b: + b.alter_column('owner', server_default=None) + b.alter_column('time_completed', server_default=None) + b.alter_column('time_modified', server_default=None) + b.alter_column('time_submitted', server_default=None) + + +def downgrade(): + # Thanks again! + with op.batch_alter_table('module_builds') as b: + b.drop_column('time_submitted') + b.drop_column('time_modified') + b.drop_column('time_completed') + b.drop_column('owner') diff --git a/rida/models.py b/rida/models.py index c2b6b677..c296e205 100644 --- a/rida/models.py +++ b/rida/models.py @@ -25,12 +25,11 @@ """ SQLAlchemy Database models for the Flask app """ - -from rida import db, log +from datetime import datetime from sqlalchemy.orm import validates - import modulemd as _modulemd +from rida import db, log import rida.messaging @@ -83,6 +82,10 @@ class ModuleBuild(RidaBase): modulemd = db.Column(db.String, nullable=False) koji_tag = db.Column(db.String) # This gets set after 'wait' scmurl = db.Column(db.String) + owner = db.Column(db.String, nullable=False) + time_submitted = db.Column(db.DateTime, nullable=False) + time_modified = db.Column(db.DateTime) + time_completed = db.Column(db.DateTime) # A monotonically increasing integer that represents which batch or # iteration this module is currently on for successive rebuilds of its @@ -125,7 +128,8 @@ class ModuleBuild(RidaBase): return session.query(cls).filter(cls.id==event['msg']['id']).first() @classmethod - def create(cls, session, conf, name, version, release, modulemd, scmurl): + def create(cls, session, conf, name, version, release, modulemd, scmurl, username): + now = datetime.utcnow() module = cls( name=name, version=version, @@ -133,6 +137,9 @@ class ModuleBuild(RidaBase): state="init", modulemd=modulemd, scmurl=scmurl, + owner=username, + time_submitted=now, + time_modified=now ) session.add(module) session.commit() diff --git a/rida/views.py b/rida/views.py index cb18e69b..0ccb7d0f 100644 --- a/rida/views.py +++ b/rida/views.py @@ -114,6 +114,7 @@ def submit_build(): release=mmd.release, modulemd=yaml, scmurl=url, + username=username ) def failure(message, code): From 864ba5104ea5d0ed3960fe0d3d864043eb51e1ec Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Wed, 17 Aug 2016 10:30:01 -0400 Subject: [PATCH 05/10] Modify the database timestamps when the build state changes --- rida/models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rida/models.py b/rida/models.py index c296e205..847339a0 100644 --- a/rida/models.py +++ b/rida/models.py @@ -138,8 +138,7 @@ class ModuleBuild(RidaBase): modulemd=modulemd, scmurl=scmurl, owner=username, - time_submitted=now, - time_modified=now + time_submitted=now ) session.add(module) session.commit() @@ -153,8 +152,14 @@ class ModuleBuild(RidaBase): def transition(self, conf, state): """ Record that a build has transitioned state. """ + now = datetime.utcnow() old_state = self.state self.state = state + self.time_modified = now + + if self.state in ['done', 'failed']: + self.time_completed = now + log.debug("%r, state %r->%r" % (self, old_state, self.state)) rida.messaging.publish( modname='rida', From c79139b82c19b406cec5f536755709a73227b178 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Wed, 17 Aug 2016 10:36:46 -0400 Subject: [PATCH 06/10] Add additional info to the module-build(s) API output --- rida/models.py | 30 ++++++++++++++++++++++++++++++ rida/views.py | 10 ++-------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/rida/models.py b/rida/models.py index 847339a0..6bb66ec6 100644 --- a/rida/models.py +++ b/rida/models.py @@ -199,11 +199,41 @@ class ModuleBuild(RidaBase): 'state': self.state, 'state_name': INVERSE_BUILD_STATES[self.state], 'scmurl': self.scmurl, + 'owner': self.owner, + 'time_submitted': self.time_submitted, + 'time_modified': self.time_modified, + 'time_completed': self.time_completed, # TODO, show their entire .json() ? 'component_builds': [build.id for build in self.component_builds], } + @staticmethod + def _utc_datetime_to_iso(datetime_object): + """ + Takes a UTC datetime object and returns an ISO formatted string + :param datetime_object: datetime.datetime + :return: string with datetime in ISO format + """ + if datetime_object: + # Converts the datetime to ISO 8601 + return datetime_object.strftime("%Y-%m-%dT%H:%M:%SZ") + + return None + + def api_json(self): + + return { + "id": self.id, + "state": self.state, + "owner": self.owner, + "name": self.name, + "time_submitted": self._utc_datetime_to_iso(self.time_submitted), + "time_modified": self._utc_datetime_to_iso(self.time_modified), + "time_completed": self._utc_datetime_to_iso(self.time_completed), + "tasks": self.tasks() + } + def tasks(self): """ :return: dictionary containing the tasks associated with the build diff --git a/rida/views.py b/rida/views.py index 0ccb7d0f..a032e108 100644 --- a/rida/views.py +++ b/rida/views.py @@ -175,8 +175,7 @@ def query_builds(): } if verbose_flag.lower() == 'true' or verbose_flag == '1': - json_data['items'] = [{'id': item.id, 'state': item.state, 'tasks': item.tasks()} - for item in p_query.items] + json_data['items'] = [item.api_json() for item in p_query.items] else: json_data['items'] = [{'id': item.id, 'state': item.state} for item in p_query.items] @@ -189,11 +188,6 @@ def query_build(id): module = models.ModuleBuild.query.filter_by(id=id).first() if module: - - return jsonify({ - "id": module.id, - "state": module.state, - "tasks": module.tasks() - }), 200 + return jsonify(module.api_json()), 200 else: return "No such module found.", 404 From 361de946934ec78ead6892911baca9260773151b Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Wed, 17 Aug 2016 10:37:24 -0400 Subject: [PATCH 07/10] Add filtering to the module-builds route --- rida/errors.py | 25 +++++++++++++++++++++ rida/utils.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ rida/views.py | 14 +++++++----- 3 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 rida/errors.py diff --git a/rida/errors.py b/rida/errors.py new file mode 100644 index 00000000..f44c4d99 --- /dev/null +++ b/rida/errors.py @@ -0,0 +1,25 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Matt Prahl +""" Defines custom exceptions and error handling functions """ + +class ValidationError(ValueError): + pass diff --git a/rida/utils.py b/rida/utils.py index 3a54c216..caa02f65 100644 --- a/rida/utils.py +++ b/rida/utils.py @@ -22,9 +22,12 @@ # Matt Prahl """ Utility functions for rida. """ from flask import request, url_for +from datetime import datetime +import re import functools import time from rida import log, models +from errors import ValidationError def retry(timeout=120, interval=30, wait_on=Exception): @@ -96,3 +99,61 @@ def pagination_metadata(p_query): per_page=p_query.per_page, _external=True) return pagination_data + + +def filter_module_builds(flask_request): + """ + Returns a flask_sqlalchemy.Pagination object based on the request parameters + :param request: Flask request object + :return: flask_sqlalchemy.Pagination + """ + search_query = dict() + state = flask_request.args.get('state', None) + + if state: + if state.isdigit(): + search_query['state'] = state + else: + if state in models.BUILD_STATES: + search_query['state'] = models.BUILD_STATES[state] + else: + raise ValidationError('An invalid state was supplied') + + for key in ['name', 'owner']: + if flask_request.args.get(key, None): + search_query[key] = flask_request.args[key] + + query = models.ModuleBuild.query + + if search_query: + query = query.filter_by(**search_query) + + # This is used when filtering the date request parameters, but it is here to avoid recompiling + utc_iso_datetime_regex = re.compile(r'^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?' + r'(?:Z|[-+]00(?::00)?)?$') + + # Filter the query based on date request parameters + for item in ('submitted', 'modified', 'completed'): + for context in ('before', 'after'): + request_arg = '%s_%s' % (item, context) # i.e. submitted_before + iso_datetime_arg = request.args.get(request_arg, None) + + if iso_datetime_arg: + iso_datetime_matches = re.match(utc_iso_datetime_regex, iso_datetime_arg) + + if not iso_datetime_matches or not iso_datetime_matches.group('datetime'): + raise ValidationError('An invalid Zulu ISO 8601 timestamp was provided for the "%s" parameter' + % request_arg) + # Converts the ISO 8601 string to a datetime object for SQLAlchemy to use to filter + item_datetime = datetime.strptime(iso_datetime_matches.group('datetime'), '%Y-%m-%dT%H:%M:%S') + # Get the database column to filter against + column = getattr(models.ModuleBuild, 'time_' + item) + + if context == 'after': + query = query.filter(column >= item_datetime) + elif context == 'before': + query = query.filter(column <= item_datetime) + + page = flask_request.args.get('page', 1, type=int) + per_page = flask_request.args.get('per_page', 10, type=int) + return query.paginate(page, per_page, False) diff --git a/rida/views.py b/rida/views.py index a032e108..0921309d 100644 --- a/rida/views.py +++ b/rida/views.py @@ -39,13 +39,13 @@ import shutil import tempfile from rida import app, conf, db, log from rida import models -from rida.utils import pagination_metadata +from rida.utils import pagination_metadata, filter_module_builds +from errors import ValidationError @app.route("/rida/module-builds/", methods=["POST"]) def submit_build(): """Handles new module build submissions.""" - # Get the time from when the build was submitted username = rida.auth.is_packager(conf.pkgdb_api_url) if not username: return "You must use your Fedora certificate when submitting a new build", 403 @@ -165,15 +165,17 @@ def submit_build(): @app.route("/rida/module-builds/", methods=["GET"]) def query_builds(): """Lists all tracked module builds.""" - page = request.args.get('page', 1, type=int) - per_page = request.args.get('per_page', 10, type=int) - p_query = models.ModuleBuild.query.paginate(page, per_page, False) - verbose_flag = request.args.get('verbose', 'false') + try: + p_query = filter_module_builds(request) + except ValidationError as e: + return e.message, 400 json_data = { 'meta': pagination_metadata(p_query) } + verbose_flag = request.args.get('verbose', 'false') + if verbose_flag.lower() == 'true' or verbose_flag == '1': json_data['items'] = [item.api_json() for item in p_query.items] else: From 4997e062581cce2c7a016d31e42ea70e355b823a Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Mon, 22 Aug 2016 10:29:58 -0400 Subject: [PATCH 08/10] Update Vagrant to use Fedora 24 --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index bc13b756..bb10a555 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,7 +13,7 @@ $script = <