From 13b7bcd03ea19158e12cbc30bbd27b0ce7990313 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Wed, 15 Feb 2017 14:46:10 +0100 Subject: [PATCH] Support for state tracing of modules and components - DB migration scripts. - Models: ComponentBuildTrace, ModuleBuildTrace. - ModuleBuild.state_trace method for querying for a particular module's state history. - SQLAlchemy before commit session event handler for recording module/component state changes. - REST API verbose mode for getting state trace of a particular module. - Tests use make_session, so that event handlers are in effect. - Short info in README about verbose mode. - Tests verifying whether state trace information about a module appears in verbose mode. - Other minor fixes (RidaBase -> MBSBase, PEP8...) --- README.rst | 10 +- module_build_service/__init__.py | 2 +- .../335455a30585_state_reason_history.py | 40 +++++++ ...697622859_add_optional_columns_for_copr.py | 2 +- module_build_service/models.py | 106 +++++++++++++++++- module_build_service/views.py | 10 +- tests/__init__.py | 58 +++++----- tests/test_models/test_models.py | 31 ++++- tests/test_views/test_views.py | 40 +++++++ 9 files changed, 256 insertions(+), 43 deletions(-) create mode 100644 module_build_service/migrations/versions/335455a30585_state_reason_history.py diff --git a/README.rst b/README.rst index 87459e54..adae2269 100644 --- a/README.rst +++ b/README.rst @@ -81,7 +81,8 @@ about the referenced build task. "tasks": { "rpms/foo" : "6378/closed", "rpms/bar : "6379/open" - } + }, + ... } "id" is the ID of the task. "state" refers to the orchestrator module @@ -90,6 +91,13 @@ build state and might be one of "init", "wait", "build", "done", "failed" or "type/NVR" and related koji or other supported buildsystem tasks and their states. +By adding ``?verbose=1`` to the request, additional detailed information +about the module can be obtained. + +:: + + GET /module-build-service/1/module-builds/42?verbose=1 + Listing all module builds ------------------------- diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index 412393ac..36516682 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -42,10 +42,10 @@ for a number of tasks: from flask import Flask, has_app_context, url_for from flask_sqlalchemy import SQLAlchemy +from logging import getLogger from module_build_service.logger import init_logging -from logging import getLogger from module_build_service.errors import ( ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound, Forbidden, json_error) diff --git a/module_build_service/migrations/versions/335455a30585_state_reason_history.py b/module_build_service/migrations/versions/335455a30585_state_reason_history.py new file mode 100644 index 00000000..0fe72394 --- /dev/null +++ b/module_build_service/migrations/versions/335455a30585_state_reason_history.py @@ -0,0 +1,40 @@ +"""state reason history + +Revision ID: 335455a30585 +Revises: 474697622859 +Create Date: 2017-02-10 11:38:55.249234 + +""" + +# revision identifiers, used by Alembic. +revision = '335455a30585' +down_revision = '474697622859' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'module_builds_trace', + sa.Column('id', sa.Integer(), primary_key=True), + sa.Column('module_id', sa.Integer(), sa.ForeignKey('module_builds.id'), nullable=False), + sa.Column('state_time', sa.DateTime(), nullable=False), + sa.Column('state', sa.Integer(), nullable=True), + sa.Column('state_reason', sa.String(), nullable=True) + ) + + op.create_table( + 'component_builds_trace', + sa.Column('id', sa.Integer(), primary_key=True), + sa.Column('component_id', sa.Integer(), sa.ForeignKey('component_builds.id'), nullable=False), + sa.Column('state_time', sa.DateTime(), nullable=False), + sa.Column('state', sa.Integer(), nullable=True), + sa.Column('state_reason', sa.String(), nullable=True), + sa.Column('task_id', sa.Integer(), nullable=True), + ) + + +def downgrade(): + op.drop_table('module_builds_trace') + op.drop_table('component_builds_trace') diff --git a/module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py b/module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py index c3d6b5b4..d8426cb0 100644 --- a/module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py +++ b/module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py @@ -1,7 +1,7 @@ """Add optional columns for copr Revision ID: 474697622859 -Revises: 0ef60c3ed440 +Revises: a1fc0736bca8 Create Date: 2017-02-21 11:18:22.304038 """ diff --git a/module_build_service/models.py b/module_build_service/models.py index cffc9a8d..e024b28c 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -29,7 +29,7 @@ import contextlib from datetime import datetime -from sqlalchemy import engine_from_config, or_ +from sqlalchemy import engine_from_config, event from sqlalchemy.orm import validates, scoped_session, sessionmaker import modulemd as _modulemd @@ -75,6 +75,7 @@ def make_session(conf): 'sqlalchemy.url': conf.sqlalchemy_database_uri, }) session = scoped_session(sessionmaker(bind=engine))() + event.listen(session, "before_commit", session_before_commit_handlers) try: yield session session.commit() @@ -86,12 +87,12 @@ def make_session(conf): session.close() -class RidaBase(db.Model): +class MBSBase(db.Model): # TODO -- we can implement functionality here common to all our model classes __abstract__ = True -class ModuleBuild(RidaBase): +class ModuleBuild(MBSBase): __tablename__ = "module_builds" id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String, nullable=False) @@ -195,6 +196,12 @@ class ModuleBuild(RidaBase): if state_reason: self.state_reason = state_reason + # record module's state change + mbt = ModuleBuildTrace(state_time=now, + state=self.state, + state_reason=state_reason) + self.module_builds_trace.append(mbt) + log.debug("%r, state %r->%r" % (self, old_state, self.state)) if old_state != self.state: module_build_service.messaging.publish( @@ -217,8 +224,8 @@ class ModuleBuild(RidaBase): """ tag = event.repo_tag.strip('-build') query = session.query(cls)\ - .filter(cls.koji_tag==tag)\ - .filter(cls.state==BUILD_STATES["build"]) + .filter(cls.koji_tag == tag)\ + .filter(cls.state == BUILD_STATES["build"]) count = query.count() if count > 1: @@ -241,9 +248,16 @@ class ModuleBuild(RidaBase): 'time_submitted': self.time_submitted, 'time_modified': self.time_modified, 'time_completed': self.time_completed, + "tasks": self.tasks(), # TODO, show their entire .json() ? 'component_builds': [build.id for build in self.component_builds], 'modulemd': self.modulemd, + 'state_trace': [{'time': record.state_time, + 'state': record.state, + 'state_name': INVERSE_BUILD_STATES[record.state], + 'reason': record.state_reason} + for record + in self.state_trace(self.id)] } @staticmethod @@ -285,13 +299,43 @@ class ModuleBuild(RidaBase): return tasks + def state_trace(self, module_id): + return ModuleBuildTrace.query.filter_by( + module_id=module_id).order_by(ModuleBuildTrace.state_time).all() + def __repr__(self): return "" % ( self.name, self.stream, self.version, INVERSE_BUILD_STATES[self.state], self.batch, self.state_reason) -class ComponentBuild(RidaBase): +class ModuleBuildTrace(MBSBase): + __tablename__ = "module_builds_trace" + id = db.Column(db.Integer, primary_key=True) + module_id = db.Column(db.Integer, db.ForeignKey('module_builds.id'), nullable=False) + state_time = db.Column(db.DateTime, nullable=False) + state = db.Column(db.Integer, nullable=True) + state_reason = db.Column(db.String, nullable=True) + + module_build = db.relationship('ModuleBuild', backref='module_builds_trace', lazy=False) + + def json(self): + retval = { + 'id': self.id, + 'module_id': self.module_id, + 'state_time': self.state_time, + 'state': self.state, + 'state_reason': self.state_reason, + } + + return retval + + def __repr__(self): + return "" % ( + self.id, self.module_id, self.state_time, self.state, self.state_reason) + + +class ComponentBuild(MBSBase): __tablename__ = "component_builds" id = db.Column(db.Integer, primary_key=True) package = db.Column(db.String, nullable=False) @@ -361,3 +405,53 @@ class ComponentBuild(RidaBase): def __repr__(self): return "" % ( self.package, self.module_id, self.state, self.task_id, self.batch, self.state_reason) + + +class ComponentBuildTrace(MBSBase): + __tablename__ = "component_builds_trace" + id = db.Column(db.Integer, primary_key=True) + component_id = db.Column(db.Integer, db.ForeignKey('component_builds.id'), nullable=False) + state_time = db.Column(db.DateTime, nullable=False) + state = db.Column(db.Integer, nullable=True) + state_reason = db.Column(db.String, nullable=True) + task_id = db.Column(db.Integer, nullable=True) + + component_build = db.relationship('ComponentBuild', backref='component_builds_trace', lazy=False) + + def json(self): + retval = { + 'id': self.id, + 'component_id': self.component_id, + 'state_time': self.state_time, + 'state': self.state, + 'state_reason': self.state_reason, + 'task_id': self.task_id, + } + + return retval + + def __repr__(self): + return "" % ( + self.id, self.component_id, self.state_time, self.state, self.state_reason, self.task_id) + + +def session_before_commit_handlers(session): + # new and updated items + for item in (set(session.new) | set(session.dirty)): + + # handlers for module builds + if isinstance(item, ModuleBuild): + mbt = ModuleBuildTrace( + state_time=datetime.utcnow(), + state=item.state, + state_reason=item.state_reason) + item.module_builds_trace.append(mbt) + + # handlers for component builds + elif isinstance(item, ComponentBuild): + cbt = ComponentBuildTrace( + state_time=datetime.utcnow(), + state=item.state, + state_reason=item.state_reason, + task_id=item.task_id) + item.component_builds_trace.append(cbt) diff --git a/module_build_service/views.py b/module_build_service/views.py index cf414deb..02a47485 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -61,9 +61,12 @@ api_v1 = { }, } + class ModuleBuildAPI(MethodView): def get(self, id): + verbose_flag = request.args.get('verbose', 'false') + if id is None: # Lists all tracked module builds p_query = filter_module_builds(request) @@ -72,8 +75,6 @@ class ModuleBuildAPI(MethodView): 'meta': pagination_metadata(p_query) } - verbose_flag = request.args.get('verbose', 'false') - if verbose_flag.lower() == 'true' or verbose_flag == '1': json_data['items'] = [item.api_json() for item in p_query.items] else: @@ -86,7 +87,10 @@ class ModuleBuildAPI(MethodView): module = models.ModuleBuild.query.filter_by(id=id).first() if module: - return jsonify(module.api_json()), 200 + if verbose_flag.lower() == 'true' or verbose_flag == '1': + return jsonify(module.json()), 200 + else: + return jsonify(module.api_json()), 200 else: raise NotFound('No such module found.') diff --git a/tests/__init__.py b/tests/__init__.py index 57991206..be4a91d9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -27,7 +27,7 @@ import module_build_service from datetime import datetime, timedelta from module_build_service import db from module_build_service.config import init_config -from module_build_service.models import ModuleBuild, ComponentBuild +from module_build_service.models import ModuleBuild, ComponentBuild, make_session import modulemd from module_build_service.utils import get_scm_url_re import module_build_service.pdc @@ -39,6 +39,7 @@ def init_data(): db.session.remove() db.drop_all() db.create_all() + db.session.commit() for index in range(10): build_one = ModuleBuild() build_one.name = 'nginx' @@ -169,22 +170,24 @@ def init_data(): component_two_build_three.batch = 1 component_two_build_three.module_id = 3 + index * 3 - db.session.add(build_one) - db.session.add(component_one_build_one) - db.session.add(component_two_build_one) - db.session.add(component_one_build_two) - db.session.add(component_two_build_two) - db.session.add(component_one_build_three) - db.session.add(component_two_build_three) - db.session.add(build_two) - db.session.add(build_three) - db.session.commit() + with make_session(conf) as session: + session.add(build_one) + session.add(component_one_build_one) + session.add(component_two_build_one) + session.add(component_one_build_two) + session.add(component_two_build_two) + session.add(component_one_build_three) + session.add(component_two_build_three) + session.add(build_two) + session.add(build_three) + session.commit() def scheduler_init_data(): db.session.remove() db.drop_all() db.create_all() + db.session.commit() current_dir = os.path.dirname(__file__) star_command_yml_path = os.path.join( @@ -232,16 +235,18 @@ def scheduler_init_data(): component_two_build_one.batch = 2 component_two_build_one.module_id = 1 - db.session.add(build_one) - db.session.add(component_one_build_one) - db.session.add(component_two_build_one) - db.session.commit() + with make_session(conf) as session: + session.add(build_one) + session.add(component_one_build_one) + session.add(component_two_build_one) + session.commit() def test_resuse_component_init_data(): db.session.remove() db.drop_all() db.create_all() + db.session.commit() current_dir = os.path.dirname(__file__) formatted_testmodule_yml_path = os.path.join( @@ -379,14 +384,15 @@ def test_resuse_component_init_data(): component_four_build_two.batch = 1 component_four_build_two.module_id = 2 - db.session.add(build_one) - db.session.add(component_one_build_one) - db.session.add(component_two_build_one) - db.session.add(component_three_build_one) - db.session.add(component_four_build_one) - db.session.add(build_two) - db.session.add(component_one_build_two) - db.session.add(component_two_build_two) - db.session.add(component_three_build_two) - db.session.add(component_four_build_two) - db.session.commit() + with make_session(conf) as session: + session.add(build_one) + session.add(component_one_build_one) + session.add(component_two_build_one) + session.add(component_three_build_one) + session.add(component_four_build_one) + session.add(build_two) + session.add(component_one_build_two) + session.add(component_two_build_two) + session.add(component_three_build_two) + session.add(component_four_build_two) + session.commit() diff --git a/tests/test_models/test_models.py b/tests/test_models/test_models.py index bafee720..8851abec 100644 --- a/tests/test_models/test_models.py +++ b/tests/test_models/test_models.py @@ -20,17 +20,38 @@ # # Written by Ralph Bean -from nose.tools import eq_ - import unittest -from tests.test_models import init_data, db - -from module_build_service import models +from tests.test_models import init_data +from module_build_service import conf +from module_build_service.models import ComponentBuild, make_session class TestModels(unittest.TestCase): def setUp(self): init_data() + def test_app_sqlalchemy_events(self): + with make_session(conf) as session: + component_build = ComponentBuild() + component_build.package = 'before_models_committed' + component_build.scmurl = \ + ('git://pkgs.domain.local/rpms/before_models_committed?' + '#9999999999999999999999999999999999999999') + component_build.format = 'rpms' + component_build.task_id = 999999999 + component_build.state = 1 + component_build.nvr = 'before_models_committed-0.0.0-0.module_before_models_committed_0_0' + component_build.batch = 1 + component_build.module_id = 1 + session.add(component_build) + session.commit() + + with make_session(conf) as session: + c = session.query(ComponentBuild).filter(ComponentBuild.id == 1).one() + self.assertEquals(c.component_builds_trace[0].id, 1) + self.assertEquals(c.component_builds_trace[0].component_id, 1) + self.assertEquals(c.component_builds_trace[0].state, 1) + self.assertEquals(c.component_builds_trace[0].state_reason, None) + self.assertEquals(c.component_builds_trace[0].task_id, 999999999) diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index e90cabba..84d3fd45 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -116,6 +116,36 @@ class TestViews(unittest.TestCase): self.assertEquals(data['time_modified'], '2016-09-03T11:25:32Z') self.assertEquals(data['time_submitted'], '2016-09-03T11:23:20Z') + def test_query_build_with_verbose_mode(self): + rv = self.client.get('/module-build-service/1/module-builds/1?verbose=1') + data = json.loads(rv.data) + self.assertEquals(data['component_builds'], [1, 2]) + self.assertEquals(data['id'], 1) + self.assertEquals(data['modulemd'], '') + self.assertEquals(data['name'], 'nginx') + self.assertEquals(data['owner'], 'Moe Szyslak') + self.assertEquals(data['scmurl'], + ('git://pkgs.domain.local/modules/nginx' + '?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9')) + self.assertEquals(data['state'], 3) + self.assertEquals(data['state_name'], 'done') + self.assertEquals(data['state_reason'], None) + self.assertEquals(data['state_trace'][0]['reason'], None) + self.assertTrue(data['state_trace'][0]['time'] is not None) + self.assertEquals(data['state_trace'][0]['state'], 3) + self.assertEquals(data['state_trace'][0]['state_name'], 'done') + self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/1') + self.assertEquals(data['stream'], '1') + self.assertEquals(data['tasks'], { + 'rpms/module-build-macros': '12312321/1', + 'rpms/nginx': '12312345/1' + } + ) + self.assertEquals(data['time_completed'], u'Sat, 03 Sep 2016 11:25:32 GMT') + self.assertEquals(data['time_modified'], u'Sat, 03 Sep 2016 11:25:32 GMT') + self.assertEquals(data['time_submitted'], u'Sat, 03 Sep 2016 11:23:20 GMT') + self.assertEquals(data['version'], '2') + def test_pagination_metadata(self): rv = self.client.get('/module-build-service/1/module-builds/?per_page=8&page=2') meta_data = json.loads(rv.data)['meta'] @@ -248,6 +278,16 @@ class TestViews(unittest.TestCase): self.assertEquals(data['id'], 31) self.assertEquals(data['state_name'], 'wait') self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31') + self.assertEquals(data['state_trace'][0]['reason'], None) + self.assertTrue(data['state_trace'][0]['time'] is not None) + self.assertEquals(data['state_trace'][0]['state'], 1) + self.assertEquals(data['state_trace'][0]['state_name'], 'wait') + self.assertEquals(data['tasks'], { + u'rpms/perl-List-Compare': u'None/None', + u'rpms/perl-Tangerine': u'None/None', + u'rpms/tangerine': u'None/None' + } + ) mmd = _modulemd.ModuleMetadata() mmd.loads(data["modulemd"])