modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-03 05:03:43 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support for state tracing of modules and components

Browse Source 
- DB migration scripts.
- Models: ComponentBuildTrace, ModuleBuildTrace.
- ModuleBuild.state_trace method for querying for a particular module's
state history.
- SQLAlchemy before commit session event handler for recording
module/component state changes.
- REST API verbose mode for getting state trace of a particular module.
- Tests use make_session, so that event handlers are in effect.
- Short info in README about verbose mode.
- Tests verifying whether state trace information about a module appears
in verbose mode.
- Other minor fixes (RidaBase -> MBSBase, PEP8...)
This commit is contained in:
Filip Valder
2017-02-15 14:46:10 +01:00
parent 7dfb647e08
commit 13b7bcd03e
9 changed files with 256 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
README.rst
View File

@@ -81,7 +81,8 @@ about the referenced build task.
"tasks": {
"rpms/foo" : "6378/closed",
"rpms/bar : "6379/open"
}
},
...
}
"id" is the ID of the task. "state" refers to the orchestrator module
@@ -90,6 +91,13 @@ build state and might be one of "init", "wait", "build", "done", "failed" or
"type/NVR" and related koji or other supported buildsystem tasks and
their states.
By adding ``?verbose=1`` to the request, additional detailed information
about the module can be obtained.
::
GET /module-build-service/1/module-builds/42?verbose=1
Listing all module builds
-------------------------

2
module_build_service/__init__.py
View File

@@ -42,10 +42,10 @@ for a number of tasks:
from flask import Flask, has_app_context, url_for
from flask_sqlalchemy import SQLAlchemy
from logging import getLogger
from module_build_service.logger import init_logging
from logging import getLogger
from module_build_service.errors import (
ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound,
Forbidden, json_error)

40
module_build_service/migrations/versions/335455a30585_state_reason_history.py Normal file
View File

@@ -0,0 +1,40 @@
"""state reason history
Revision ID: 335455a30585
Revises: 474697622859
Create Date: 2017-02-10 11:38:55.249234
"""
# revision identifiers, used by Alembic.
revision = '335455a30585'
down_revision = '474697622859'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table(
'module_builds_trace',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column('module_id', sa.Integer(), sa.ForeignKey('module_builds.id'), nullable=False),
sa.Column('state_time', sa.DateTime(), nullable=False),
sa.Column('state', sa.Integer(), nullable=True),
sa.Column('state_reason', sa.String(), nullable=True)
)
op.create_table(
'component_builds_trace',
sa.Column('id', sa.Integer(), primary_key=True),
sa.Column('component_id', sa.Integer(), sa.ForeignKey('component_builds.id'), nullable=False),
sa.Column('state_time', sa.DateTime(), nullable=False),
sa.Column('state', sa.Integer(), nullable=True),
sa.Column('state_reason', sa.String(), nullable=True),
sa.Column('task_id', sa.Integer(), nullable=True),
)
def downgrade():
op.drop_table('module_builds_trace')
op.drop_table('component_builds_trace')

2
module_build_service/migrations/versions/474697622859_add_optional_columns_for_copr.py
View File

@@ -1,7 +1,7 @@
"""Add optional columns for copr
Revision ID: 474697622859
Revises: 0ef60c3ed440
Revises: a1fc0736bca8
Create Date: 2017-02-21 11:18:22.304038
"""

106
module_build_service/models.py
View File

@@ -29,7 +29,7 @@
import contextlib
from datetime import datetime
from sqlalchemy import engine_from_config, or_
from sqlalchemy import engine_from_config, event
from sqlalchemy.orm import validates, scoped_session, sessionmaker
import modulemd as _modulemd
@@ -75,6 +75,7 @@ def make_session(conf):
'sqlalchemy.url': conf.sqlalchemy_database_uri,
})
session = scoped_session(sessionmaker(bind=engine))()
event.listen(session, "before_commit", session_before_commit_handlers)
try:
yield session
session.commit()
@@ -86,12 +87,12 @@ def make_session(conf):
session.close()
class RidaBase(db.Model):
class MBSBase(db.Model):
# TODO -- we can implement functionality here common to all our model classes
__abstract__ = True
class ModuleBuild(RidaBase):
class ModuleBuild(MBSBase):
__tablename__ = "module_builds"
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String, nullable=False)
@@ -195,6 +196,12 @@ class ModuleBuild(RidaBase):
if state_reason:
self.state_reason = state_reason
# record module's state change
mbt = ModuleBuildTrace(state_time=now,
state=self.state,
state_reason=state_reason)
self.module_builds_trace.append(mbt)
log.debug("%r, state %r->%r" % (self, old_state, self.state))
if old_state != self.state:
module_build_service.messaging.publish(
@@ -217,8 +224,8 @@ class ModuleBuild(RidaBase):
"""
tag = event.repo_tag.strip('-build')
query = session.query(cls)\
.filter(cls.koji_tag==tag)\
.filter(cls.state==BUILD_STATES["build"])
.filter(cls.koji_tag == tag)\
.filter(cls.state == BUILD_STATES["build"])
count = query.count()
if count > 1:
@@ -241,9 +248,16 @@ class ModuleBuild(RidaBase):
'time_submitted': self.time_submitted,
'time_modified': self.time_modified,
'time_completed': self.time_completed,
"tasks": self.tasks(),
# TODO, show their entire .json() ?
'component_builds': [build.id for build in self.component_builds],
'modulemd': self.modulemd,
'state_trace': [{'time': record.state_time,
'state': record.state,
'state_name': INVERSE_BUILD_STATES[record.state],
'reason': record.state_reason}
for record
in self.state_trace(self.id)]
}
@staticmethod
@@ -285,13 +299,43 @@ class ModuleBuild(RidaBase):
return tasks
def state_trace(self, module_id):
return ModuleBuildTrace.query.filter_by(
module_id=module_id).order_by(ModuleBuildTrace.state_time).all()
def __repr__(self):
return "<ModuleBuild %s, stream=%s, version=%s, state %r, batch %r, state_reason %r>" % (
self.name, self.stream, self.version,
INVERSE_BUILD_STATES[self.state], self.batch, self.state_reason)
class ComponentBuild(RidaBase):
class ModuleBuildTrace(MBSBase):
__tablename__ = "module_builds_trace"
id = db.Column(db.Integer, primary_key=True)
module_id = db.Column(db.Integer, db.ForeignKey('module_builds.id'), nullable=False)
state_time = db.Column(db.DateTime, nullable=False)
state = db.Column(db.Integer, nullable=True)
state_reason = db.Column(db.String, nullable=True)
module_build = db.relationship('ModuleBuild', backref='module_builds_trace', lazy=False)
def json(self):
retval = {
'id': self.id,
'module_id': self.module_id,
'state_time': self.state_time,
'state': self.state,
'state_reason': self.state_reason,
}
return retval
def __repr__(self):
return "<ModuleBuildTrace %s, module_id: %s, state_time: %r, state: %s, state_reason: %s>" % (
self.id, self.module_id, self.state_time, self.state, self.state_reason)
class ComponentBuild(MBSBase):
__tablename__ = "component_builds"
id = db.Column(db.Integer, primary_key=True)
package = db.Column(db.String, nullable=False)
@@ -361,3 +405,53 @@ class ComponentBuild(RidaBase):
def __repr__(self):
return "<ComponentBuild %s, %r, state: %r, task_id: %r, batch: %r, state_reason: %s>" % (
self.package, self.module_id, self.state, self.task_id, self.batch, self.state_reason)
class ComponentBuildTrace(MBSBase):
__tablename__ = "component_builds_trace"
id = db.Column(db.Integer, primary_key=True)
component_id = db.Column(db.Integer, db.ForeignKey('component_builds.id'), nullable=False)
state_time = db.Column(db.DateTime, nullable=False)
state = db.Column(db.Integer, nullable=True)
state_reason = db.Column(db.String, nullable=True)
task_id = db.Column(db.Integer, nullable=True)
component_build = db.relationship('ComponentBuild', backref='component_builds_trace', lazy=False)
def json(self):
retval = {
'id': self.id,
'component_id': self.component_id,
'state_time': self.state_time,
'state': self.state,
'state_reason': self.state_reason,
'task_id': self.task_id,
}
return retval
def __repr__(self):
return "<ComponentBuildTrace %s, component_id: %s, state_time: %r, state: %s, state_reason: %s, task_id: %s>" % (
self.id, self.component_id, self.state_time, self.state, self.state_reason, self.task_id)
def session_before_commit_handlers(session):
# new and updated items
for item in (set(session.new) | set(session.dirty)):
# handlers for module builds
if isinstance(item, ModuleBuild):
mbt = ModuleBuildTrace(
state_time=datetime.utcnow(),
state=item.state,
state_reason=item.state_reason)
item.module_builds_trace.append(mbt)
# handlers for component builds
elif isinstance(item, ComponentBuild):
cbt = ComponentBuildTrace(
state_time=datetime.utcnow(),
state=item.state,
state_reason=item.state_reason,
task_id=item.task_id)
item.component_builds_trace.append(cbt)

10
module_build_service/views.py
View File

@@ -61,9 +61,12 @@ api_v1 = {
},
}
class ModuleBuildAPI(MethodView):
def get(self, id):
verbose_flag = request.args.get('verbose', 'false')
if id is None:
# Lists all tracked module builds
p_query = filter_module_builds(request)
@@ -72,8 +75,6 @@ class ModuleBuildAPI(MethodView):
'meta': pagination_metadata(p_query)
}
verbose_flag = request.args.get('verbose', 'false')
if verbose_flag.lower() == 'true' or verbose_flag == '1':
json_data['items'] = [item.api_json() for item in p_query.items]
else:
@@ -86,7 +87,10 @@ class ModuleBuildAPI(MethodView):
module = models.ModuleBuild.query.filter_by(id=id).first()
if module:
return jsonify(module.api_json()), 200
if verbose_flag.lower() == 'true' or verbose_flag == '1':
return jsonify(module.json()), 200
else:
return jsonify(module.api_json()), 200
else:
raise NotFound('No such module found.')

58
tests/__init__.py
View File

@@ -27,7 +27,7 @@ import module_build_service
from datetime import datetime, timedelta
from module_build_service import db
from module_build_service.config import init_config
from module_build_service.models import ModuleBuild, ComponentBuild
from module_build_service.models import ModuleBuild, ComponentBuild, make_session
import modulemd
from module_build_service.utils import get_scm_url_re
import module_build_service.pdc
@@ -39,6 +39,7 @@ def init_data():
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
for index in range(10):
build_one = ModuleBuild()
build_one.name = 'nginx'
@@ -169,22 +170,24 @@ def init_data():
component_two_build_three.batch = 1
component_two_build_three.module_id = 3 + index * 3
db.session.add(build_one)
db.session.add(component_one_build_one)
db.session.add(component_two_build_one)
db.session.add(component_one_build_two)
db.session.add(component_two_build_two)
db.session.add(component_one_build_three)
db.session.add(component_two_build_three)
db.session.add(build_two)
db.session.add(build_three)
db.session.commit()
with make_session(conf) as session:
session.add(build_one)
session.add(component_one_build_one)
session.add(component_two_build_one)
session.add(component_one_build_two)
session.add(component_two_build_two)
session.add(component_one_build_three)
session.add(component_two_build_three)
session.add(build_two)
session.add(build_three)
session.commit()
def scheduler_init_data():
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
current_dir = os.path.dirname(__file__)
star_command_yml_path = os.path.join(
@@ -232,16 +235,18 @@ def scheduler_init_data():
component_two_build_one.batch = 2
component_two_build_one.module_id = 1
db.session.add(build_one)
db.session.add(component_one_build_one)
db.session.add(component_two_build_one)
db.session.commit()
with make_session(conf) as session:
session.add(build_one)
session.add(component_one_build_one)
session.add(component_two_build_one)
session.commit()
def test_resuse_component_init_data():
db.session.remove()
db.drop_all()
db.create_all()
db.session.commit()
current_dir = os.path.dirname(__file__)
formatted_testmodule_yml_path = os.path.join(
@@ -379,14 +384,15 @@ def test_resuse_component_init_data():
component_four_build_two.batch = 1
component_four_build_two.module_id = 2
db.session.add(build_one)
db.session.add(component_one_build_one)
db.session.add(component_two_build_one)
db.session.add(component_three_build_one)
db.session.add(component_four_build_one)
db.session.add(build_two)
db.session.add(component_one_build_two)
db.session.add(component_two_build_two)
db.session.add(component_three_build_two)
db.session.add(component_four_build_two)
db.session.commit()
with make_session(conf) as session:
session.add(build_one)
session.add(component_one_build_one)
session.add(component_two_build_one)
session.add(component_three_build_one)
session.add(component_four_build_one)
session.add(build_two)
session.add(component_one_build_two)
session.add(component_two_build_two)
session.add(component_three_build_two)
session.add(component_four_build_two)
session.commit()

31
tests/test_models/test_models.py
View File

@@ -20,17 +20,38 @@
#
# Written by Ralph Bean <rbean@redhat.com>
from nose.tools import eq_
import unittest
from tests.test_models import init_data, db
from module_build_service import models
from tests.test_models import init_data
from module_build_service import conf
from module_build_service.models import ComponentBuild, make_session
class TestModels(unittest.TestCase):
def setUp(self):
init_data()
def test_app_sqlalchemy_events(self):
with make_session(conf) as session:
component_build = ComponentBuild()
component_build.package = 'before_models_committed'
component_build.scmurl = \
('git://pkgs.domain.local/rpms/before_models_committed?'
'#9999999999999999999999999999999999999999')
component_build.format = 'rpms'
component_build.task_id = 999999999
component_build.state = 1
component_build.nvr = 'before_models_committed-0.0.0-0.module_before_models_committed_0_0'
component_build.batch = 1
component_build.module_id = 1
session.add(component_build)
session.commit()
with make_session(conf) as session:
c = session.query(ComponentBuild).filter(ComponentBuild.id == 1).one()
self.assertEquals(c.component_builds_trace[0].id, 1)
self.assertEquals(c.component_builds_trace[0].component_id, 1)
self.assertEquals(c.component_builds_trace[0].state, 1)
self.assertEquals(c.component_builds_trace[0].state_reason, None)
self.assertEquals(c.component_builds_trace[0].task_id, 999999999)

40
tests/test_views/test_views.py
View File

@@ -116,6 +116,36 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['time_modified'], '2016-09-03T11:25:32Z')
self.assertEquals(data['time_submitted'], '2016-09-03T11:23:20Z')
def test_query_build_with_verbose_mode(self):
rv = self.client.get('/module-build-service/1/module-builds/1?verbose=1')
data = json.loads(rv.data)
self.assertEquals(data['component_builds'], [1, 2])
self.assertEquals(data['id'], 1)
self.assertEquals(data['modulemd'], '')
self.assertEquals(data['name'], 'nginx')
self.assertEquals(data['owner'], 'Moe Szyslak')
self.assertEquals(data['scmurl'],
('git://pkgs.domain.local/modules/nginx'
'?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9'))
self.assertEquals(data['state'], 3)
self.assertEquals(data['state_name'], 'done')
self.assertEquals(data['state_reason'], None)
self.assertEquals(data['state_trace'][0]['reason'], None)
self.assertTrue(data['state_trace'][0]['time'] is not None)
self.assertEquals(data['state_trace'][0]['state'], 3)
self.assertEquals(data['state_trace'][0]['state_name'], 'done')
self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/1')
self.assertEquals(data['stream'], '1')
self.assertEquals(data['tasks'], {
'rpms/module-build-macros': '12312321/1',
'rpms/nginx': '12312345/1'
}
)
self.assertEquals(data['time_completed'], u'Sat, 03 Sep 2016 11:25:32 GMT')
self.assertEquals(data['time_modified'], u'Sat, 03 Sep 2016 11:25:32 GMT')
self.assertEquals(data['time_submitted'], u'Sat, 03 Sep 2016 11:23:20 GMT')
self.assertEquals(data['version'], '2')
def test_pagination_metadata(self):
rv = self.client.get('/module-build-service/1/module-builds/?per_page=8&page=2')
meta_data = json.loads(rv.data)['meta']
@@ -248,6 +278,16 @@ class TestViews(unittest.TestCase):
self.assertEquals(data['id'], 31)
self.assertEquals(data['state_name'], 'wait')
self.assertEquals(data['state_url'], '/module-build-service/1/module-builds/31')
self.assertEquals(data['state_trace'][0]['reason'], None)
self.assertTrue(data['state_trace'][0]['time'] is not None)
self.assertEquals(data['state_trace'][0]['state'], 1)
self.assertEquals(data['state_trace'][0]['state_name'], 'wait')
self.assertEquals(data['tasks'], {
u'rpms/perl-List-Compare': u'None/None',
u'rpms/perl-Tangerine': u'None/None',
u'rpms/tangerine': u'None/None'
}
)
mmd = _modulemd.ModuleMetadata()
mmd.loads(data["modulemd"])