modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-07 23:33:19 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'Database-Migration-Support-and-Cleanup'

Browse Source
This commit is contained in:
Ralph Bean
2016-08-10 13:48:45 -04:00
parent ad3eb578fb 124b3c3900
commit c9aefef5f2
28 changed files with 574 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -2,3 +2,5 @@
rida.db
server.crt
server.key
.vagrant
.idea

4
README.rst
View File

@@ -137,8 +137,8 @@ _`Module Build States`
You can see the list of possible states with::
import rida
print(rida.BUILD_STATES)
from rida.models import BUILD_STATES
print(BUILD_STATES)
Here's a description of what each of them means:

19
Vagrantfile vendored Normal file
View File

@@ -0,0 +1,19 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
$script = <<SCRIPT
dnf install -y python python-virtualenv python-devel libffi-devel redhat-rpm-config openssl-devel gcc gcc-c++ koji
pip install -r /opt/fm-orchestrator/src/requirements.txt
pip install -r /opt/fm-orchestrator/src/test-requirements.txt
cd /opt/fm-orchestrator/src
python manage.py upgradedb
./generate_localhost_cert.sh
SCRIPT
Vagrant.configure("2") do |config|
config.vm.box = "box-cutter/fedora23"
config.vm.synced_folder "./", "/opt/fm-orchestrator/src"
config.vm.network "forwarded_port", guest: 5000, host: 5000
config.vm.provision "shell", inline: $script
config.vm.provision :shell, inline: "cd /opt/fm-orchestrator/src && python manage.py runssl &", run: "always"
end

56
config.py Normal file
View File

@@ -0,0 +1,56 @@
from os import path
class BaseConfiguration(object):
# Make this random (used to generate session keys)
SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0'
basedir = path.abspath(path.dirname(__file__))
SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join(basedir, 'rida.db'))
SQLALCHEMY_TRACK_MODIFICATIONS = True
# Where we should run when running "manage.py runssl" directly.
HOST = '127.0.0.1'
PORT = 5000
SYSTEM = 'koji'
MESSAGING = 'fedmsg'
KOJI_CONFIG = '/etc/rida/koji.conf'
KOJI_PROFILE = 'koji'
KOJI_ARCHES = ['x86_64']
PDC_URL = 'http://modularity.fedorainfracloud.org:8080/rest_api/v1'
PDC_INSECURE = True
PDC_DEVELOP = True
SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"]
# How often should we resort to polling, in seconds
# Set to zero to disable polling
POLLING_INTERVAL = 600
RPMS_DEFAULT_REPOSITORY = 'git://pkgs.fedoraproject.org/rpms/'
RPMS_ALLOW_REPOSITORY = False
RPMS_DEFAULT_CACHE = 'http://pkgs.fedoraproject.org/repo/pkgs/'
RPMS_ALLOW_CACHE = False
SSL_ENABLED = True
SSL_CERTIFICATE_FILE = 'server.crt'
SSL_CERTIFICATE_KEY_FILE = 'server.key'
SSL_CA_CERTIFICATE_FILE = 'cacert.pem'
PKGDB_API_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api'
# Available backends are: console, file, journal.
LOG_BACKEND = 'journal'
# Path to log file when LOG_BACKEND is set to "file".
LOG_FILE = 'rida.log'
# Available log levels are: debug, info, warn, error.
LOG_LEVEL = 'info'
class DevConfiguration(BaseConfiguration):
LOG_BACKEND = 'console'
HOST = '0.0.0.0'
class ProdConfiguration(BaseConfiguration):
pass

8
createdb.py
View File

@@ -1,8 +0,0 @@
#!/usr/bin/python3
import rida.config
import rida.database
config = rida.config.from_file("rida.conf")
rida.database.Database.create_tables(config, True)

150
manage.py Normal file
View File

@@ -0,0 +1,150 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2016 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Matt Prahl <mprahl@redhat.com> except for the test functions
from flask_script import Manager
import flask_migrate
import logging
import os
import ssl
from rida import app, conf, db
from rida.config import Config
from rida.pdc import get_pdc_client_session, get_module, get_module_runtime_dependencies, get_module_tag, \
get_module_build_dependencies
import rida.auth
manager = Manager(app)
migrate = flask_migrate.Migrate(app, db)
manager.add_command('db', flask_migrate.MigrateCommand)
def _establish_ssl_context():
if not conf.ssl_enabled:
return None
# First, do some validation of the configuration
attributes = (
'ssl_certificate_file',
'ssl_certificate_key_file',
'ssl_ca_certificate_file',
)
for attribute in attributes:
value = getattr(conf, attribute, None)
if not value:
raise ValueError("%r could not be found" % attribute)
if not os.path.exists(value):
raise OSError("%s: %s file not found." % (attribute, value))
# Then, establish the ssl context and return it
ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_ctx.load_cert_chain(conf.ssl_certificate_file,
conf.ssl_certificate_key_file)
ssl_ctx.verify_mode = ssl.CERT_OPTIONAL
ssl_ctx.load_verify_locations(cafile=conf.ssl_ca_certificate_file)
return ssl_ctx
@manager.command
def testpdc():
""" A helper function to test pdc interaction
"""
cfg = Config()
cfg.pdc_url = "http://modularity.fedorainfracloud.org:8080/rest_api/v1"
cfg.pdc_insecure = True
cfg.pdc_develop = True
pdc_session = get_pdc_client_session(cfg)
module = get_module(pdc_session, {'name': 'testmodule', 'version': '4.3.43', 'release': '1'})
if module:
print ("pdc_data=%s" % str(module))
print ("deps=%s" % get_module_runtime_dependencies(pdc_session, module))
print ("build_deps=%s" % get_module_build_dependencies(pdc_session, module))
print ("tag=%s" % get_module_tag(pdc_session, module))
else:
print ('module was not found')
@manager.command
def testbuildroot():
""" A helper function to test buildroot creation
"""
# Do a locally namespaced import here to avoid importing py2-only libs in a
# py3 runtime.
from rida.builder import KojiModuleBuilder, Builder
cfg = Config()
cfg.koji_profile = "koji"
cfg.koji_config = "/etc/rida/koji.conf"
cfg.koji_arches = ["x86_64", "i686"]
mb = KojiModuleBuilder(module="testmodule-1.0", config=cfg) # or By using Builder
# mb = Builder(module="testmodule-1.0", backend="koji", config=cfg)
resume = False
if not resume:
mb.buildroot_prep()
mb.buildroot_add_dependency(["f24"])
mb.buildroot_ready()
task_id = mb.build(artifact_name="fedora-release",
source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9")
mb.buildroot_add_artifacts(["fedora-release-24-2", ]) # just example with disttag macro
mb.buildroot_ready(artifact="fedora-release-24-2")
else:
mb.buildroot_resume()
task_id = mb.build(artifact_name="fedora-release",
source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9")
@manager.command
def upgradedb():
""" Upgrades the database schema to the latest revision
"""
flask_migrate.upgrade()
@manager.command
def runssl(host=None, port=None):
""" Runs the Flask app with the HTTPS settings configured in config.py
"""
if not host:
host = conf.host
if not port:
port = conf.port
logging.info('Starting Rida')
ssl_ctx = _establish_ssl_context()
app.run(
host=host,
port=port,
request_handler=rida.auth.ClientCertRequestHandler,
ssl_context=ssl_ctx
)
if __name__ == "__main__":
manager.run()

1
migrations/README Executable file
View File

@@ -0,0 +1 @@
Generic single-database configuration.

45
migrations/alembic.ini Normal file
View File

@@ -0,0 +1,45 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

87
migrations/env.py Executable file
View File

@@ -0,0 +1,87 @@
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
import logging
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from flask import current_app
config.set_main_option('sqlalchemy.url',
current_app.config.get('SQLALCHEMY_DATABASE_URI'))
target_metadata = current_app.extensions['migrate'].db.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(url=url)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.readthedocs.org/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

22
migrations/script.py.mako Executable file
View File

@@ -0,0 +1,22 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

57
migrations/versions/a7a553e5ca1d_.py Normal file
View File

@@ -0,0 +1,57 @@
"""empty message
Revision ID: a7a553e5ca1d
Revises: None
Create Date: 2016-08-01 16:48:23.979017
"""
# revision identifiers, used by Alembic.
revision = 'a7a553e5ca1d'
down_revision = None
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('modules',
sa.Column('name', sa.String(), nullable=False),
sa.PrimaryKeyConstraint('name')
)
op.create_table('module_builds',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('version', sa.String(), nullable=False),
sa.Column('release', sa.String(), nullable=False),
sa.Column('state', sa.Integer(), nullable=False),
sa.Column('modulemd', sa.String(), nullable=False),
sa.Column('koji_tag', sa.String(), nullable=True),
sa.Column('scmurl', sa.String(), nullable=True),
sa.Column('batch', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['name'], ['modules.name'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('component_builds',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('package', sa.String(), nullable=False),
sa.Column('scmurl', sa.String(), nullable=False),
sa.Column('format', sa.String(), nullable=False),
sa.Column('task_id', sa.Integer(), nullable=True),
sa.Column('state', sa.Integer(), nullable=True),
sa.Column('nvr', sa.String(), nullable=True),
sa.Column('batch', sa.Integer(), nullable=True),
sa.Column('module_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['module_id'], ['module_builds.id'], ),
sa.PrimaryKeyConstraint('id')
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('component_builds')
op.drop_table('module_builds')
op.drop_table('modules')
### end Alembic commands ###

4
requirements.txt
View File

@@ -2,7 +2,11 @@ flask
sqlalchemy
six
fedmsg
pdc-client
modulemd
pyOpenSSL
kobo
munch
Flask-Script
Flask-SQLAlchemy
Flask-Migrate

41
rida.conf
View File

@@ -1,41 +0,0 @@
[DEFAULT]
system = koji
messaging = fedmsg
koji_config = /etc/rida/koji.conf
# See https://fedoraproject.org/wiki/Koji/WritingKojiCode#Profiles
koji_profile = koji
koji_arches = ["x86_64"]
db = sqlite:///rida.db
pdc_url = http://modularity.fedorainfracloud.org:8080/rest_api/v1/
pdc_insecure = True
pdc_develop = True
scmurls = ["git://pkgs.stg.fedoraproject.org/modules/"]
# Where we should run when running rida.py directly.
host = 127.0.0.1
port = 5000
# How often should we resort to polling, in seconds
# Set to zero to disable polling
polling_interval = 600
rpms_default_repository = git://pkgs.stg.fedoraproject.org/rpms/
rpms_allow_repository = False
rpms_default_cache = http://pkgs.stg.fedoraproject.org/repo/pkgs/
rpms_allow_cache = False
ssl_enabled = True
ssl_certificate_file = server.crt
ssl_certificate_key_file = server.key
ssl_ca_certificate_file = cacert.pem
pkgdb_api_url = https://admin.stg.fedoraproject.org/pkgdb/api
# Available backends are: console, file, journal.
log_backend = console
# Path to log file when log_backend is set to "file".
log_file = rida.log
# Available log levels are: debug, info, warn, error.
log_level = info

24
rida/__init__.py
View File

@@ -40,5 +40,27 @@ for a number of tasks:
- Emitting bus messages about all state changes so that other
infrastructure services can pick up the work.
"""
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from os import sys
import rida.logger
from logging import getLogger
from rida.database import BUILD_STATES
app = Flask(__name__)
app.config.from_envvar("RIDA_SETTINGS", silent=True)
here = sys.path[0]
if here not in ('/usr/bin', '/bin', '/usr/local/bin'):
app.config.from_object('config.DevConfiguration')
else:
app.config.from_object('config.ProdConfiguration')
db = SQLAlchemy(app)
import rida.config
conf = rida.config.from_app_config()
rida.logger.init_logging(conf)
log = getLogger(__name__)
from rida import views

2
rida/builder.py
View File

@@ -45,10 +45,10 @@ import kobo.rpmlib
import munch
from OpenSSL.SSL import SysCallError
from rida import log
import rida.utils
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
# TODO: read defaults from rida's config
KOJI_DEFAULT_GROUPS = {

54
rida/config.py
View File

@@ -25,9 +25,6 @@
"""Configuration handler functions."""
import os.path
import json
try:
import configparser # py3
except ImportError:
@@ -35,6 +32,7 @@ except ImportError:
import six
from rida import app
from rida import logger
def asbool(value):
@@ -44,54 +42,15 @@ def asbool(value):
]
def from_file(filename=None):
"""Create the configuration instance from a file.
The file name is optional and defaults to /etc/rida/rida.conf.
:param str filename: The configuration file to load, optional.
def from_app_config():
""" Create the configuration instance from the values in app.config
"""
if filename is None:
filename = "/etc/rida/rida.conf"
if not isinstance(filename, str):
raise TypeError("The configuration filename must be a string.")
if not os.path.isfile(filename):
raise IOError("The configuration file '%s' doesn't exist." % filename)
cp = configparser.ConfigParser(allow_no_value=True)
cp.read(filename)
default = cp.defaults()
conf = Config()
conf.db = default.get("db")
conf.system = default.get("system")
conf.messaging = default.get("messaging")
conf.polling_interval = int(default.get("polling_interval"))
conf.pdc_url = default.get("pdc_url")
conf.pdc_insecure = default.get("pdc_insecure")
conf.pdc_develop = default.get("pdc_develop")
conf.koji_config = default.get("koji_config")
conf.koji_profile = default.get("koji_profile")
conf.koji_arches = json.loads(default.get("koji_arches"))
conf.scmurls = json.loads(default.get("scmurls"))
conf.rpms_default_repository = default.get("rpms_default_repository")
conf.rpms_allow_repository = asbool(default.get("rpms_allow_repository"))
conf.rpms_default_cache = default.get("rpms_default_cache")
conf.rpms_allow_cache = asbool(default.get("rpms_allow_cache"))
conf.port = default.get("port")
conf.host = default.get("host")
conf.ssl_enabled = asbool(default.get("ssl_enabled"))
conf.ssl_certificate_file = default.get("ssl_certificate_file")
conf.ssl_certificate_key_file = default.get("ssl_certificate_key_file")
conf.ssl_ca_certificate_file = default.get("ssl_ca_certificate_file")
conf.pkgdb_api_url = default.get("pkgdb_api_url")
conf.log_backend = default.get("log_backend")
conf.log_file = default.get("log_file")
conf.log_level = default.get("log_level")
for key, value in app.config.items():
setattr(conf, key.lower(), value)
return conf
class Config(object):
"""Class representing the orchestrator configuration."""
@@ -201,7 +160,6 @@ class Config(object):
def koji_config(self, s):
self._koji_config = str(s)
@property
def koji_profile(self):
"""Koji URL."""

123
rida/database.py → rida/models.py
View File

@@ -21,30 +21,18 @@
#
# Written by Petr Šabata <contyk@redhat.com>
# Ralph Bean <rbean@redhat.com>
# Matt Prahl <mprahl@redhat.com>
"""Database handler functions."""
""" SQLAlchemy Database models for the Flask app
"""
from sqlalchemy import (
Column,
Integer,
String,
ForeignKey,
create_engine,
)
from sqlalchemy.orm import (
sessionmaker,
relationship,
validates,
)
from sqlalchemy.ext.declarative import declarative_base
from rida import db, log
from sqlalchemy.orm import validates
import modulemd as _modulemd
import rida.messaging
import logging
log = logging.getLogger(__name__)
# Just like koji.BUILD_STATES, except our own codes for modules.
BUILD_STATES = {
@@ -75,75 +63,33 @@ BUILD_STATES = {
INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()}
class RidaBase(object):
# TODO -- we can implement functionality here common to all our model
# classes.
pass
class RidaBase(db.Model):
# TODO -- we can implement functionality here common to all our model classes
__abstract__ = True
Base = declarative_base(cls=RidaBase)
class Database(object):
"""Class for handling database connections."""
def __init__(self, config, debug=False):
"""Initialize the database object."""
self.engine = create_engine(config.db, echo=debug)
self._session = None # Lazilly created..
def __enter__(self):
return self.session
def __exit__(self, *args, **kwargs):
self._session.close()
self._session = None
@property
def session(self):
"""Database session object."""
if not self._session:
Session = sessionmaker(bind=self.engine)
self._session = Session()
return self._session
@classmethod
def create_tables(cls, config, debug=False):
""" Creates our tables in the database.
:arg config, config object with a 'db' URL attached to it.
ie: <engine>://<user>:<password>@<host>/<dbname>
:kwarg debug, a boolean specifying wether we should have the verbose
output of sqlalchemy or not.
:return a Database connection that can be used to query to db.
"""
engine = create_engine(config.db, echo=debug)
Base.metadata.create_all(engine)
return cls(config, debug=debug)
class Module(Base):
class Module(RidaBase):
__tablename__ = "modules"
name = Column(String, primary_key=True)
name = db.Column(db.String, primary_key=True)
class ModuleBuild(Base):
class ModuleBuild(RidaBase):
__tablename__ = "module_builds"
id = Column(Integer, primary_key=True)
name = Column(String, ForeignKey('modules.name'), nullable=False)
version = Column(String, nullable=False)
release = Column(String, nullable=False)
state = Column(Integer, nullable=False)
modulemd = Column(String, nullable=False)
koji_tag = Column(String) # This gets set after 'wait'
scmurl = Column(String)
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String, db.ForeignKey('modules.name'), nullable=False)
version = db.Column(db.String, nullable=False)
release = db.Column(db.String, nullable=False)
state = db.Column(db.Integer, nullable=False)
modulemd = db.Column(db.String, nullable=False)
koji_tag = db.Column(db.String) # This gets set after 'wait'
scmurl = db.Column(db.String)
# A monotonically increasing integer that represents which batch or
# iteration this module is currently on for successive rebuilds of its
# components. Think like 'mockchain --recurse'
batch = Column(Integer, default=0)
batch = db.Column(db.Integer, default=0)
module = relationship('Module', backref='module_builds', lazy=False)
module = db.relationship('Module', backref='module_builds', lazy=False)
def current_batch(self):
""" Returns all components of this module in the current batch. """
@@ -212,8 +158,7 @@ class ModuleBuild(Base):
@classmethod
def by_state(cls, session, state):
return session.query(rida.database.ModuleBuild)\
.filter_by(state=BUILD_STATES[state]).all()
return session.query(ModuleBuild).filter_by(state=BUILD_STATES[state]).all()
@classmethod
def from_repo_done_event(cls, session, event):
@@ -253,27 +198,27 @@ class ModuleBuild(Base):
INVERSE_BUILD_STATES[self.state], self.batch)
class ComponentBuild(Base):
class ComponentBuild(RidaBase):
__tablename__ = "component_builds"
id = Column(Integer, primary_key=True)
package = Column(String, nullable=False)
scmurl = Column(String, nullable=False)
id = db.Column(db.Integer, primary_key=True)
package = db.Column(db.String, nullable=False)
scmurl = db.Column(db.String, nullable=False)
# XXX: Consider making this a proper ENUM
format = Column(String, nullable=False)
task_id = Column(Integer) # This is the id of the build in koji
format = db.Column(db.String, nullable=False)
task_id = db.Column(db.Integer) # This is the id of the build in koji
# XXX: Consider making this a proper ENUM (or an int)
state = Column(Integer)
state = db.Column(db.Integer)
# This stays as None until the build completes.
nvr = Column(String)
nvr = db.Column(db.String)
# A monotonically increasing integer that represents which batch or
# iteration this *component* is currently in. This relates to the owning
# module's batch. This one defaults to None, which means that this
# component is not currently part of a batch.
batch = Column(Integer, default=0)
batch = db.Column(db.Integer, default=0)
module_id = Column(Integer, ForeignKey('module_builds.id'), nullable=False)
module_build = relationship('ModuleBuild', backref='component_builds', lazy=False)
module_id = db.Column(db.Integer, db.ForeignKey('module_builds.id'), nullable=False)
module_build = db.relationship('ModuleBuild', backref='component_builds', lazy=False)
@classmethod
def from_component_event(cls, session, event):
@@ -301,8 +246,6 @@ class ComponentBuild(Base):
return retval
def __repr__(self):
return "<ComponentBuild %s, %r, state: %r, task_id: %r, batch: %r>" % (
self.package, self.module_id, self.state, self.task_id, self.batch)

10
rida/scheduler/handlers/components.py
View File

@@ -26,20 +26,20 @@
import logging
import rida.builder
import rida.database
import rida.pdc
import koji
from rida import models, log
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
def _finalize(config, session, msg, state):
""" Called whenever a koji build completes or fails. """
# First, find our ModuleBuild associated with this repo, if any.
component_build = rida.database.ComponentBuild.from_component_event(session, msg)
component_build = models.ComponentBuild.from_component_event(session, msg)
nvr = "{name}-{version}-{release}".format(**msg['msg'])
if not component_build:
log.debug("We have no record of %s" % nvr)
@@ -55,7 +55,7 @@ def _finalize(config, session, msg, state):
if component_build.package == 'module-build-macros':
if state != koji.BUILD_STATES['COMPLETE']:
# If the macro build failed, then the module is doomed.
parent.transition(config, state=rida.BUILD_STATES['failed'])
parent.transition(config, state=models.BUILD_STATES['failed'])
session.commit()
return
@@ -80,7 +80,7 @@ def _finalize(config, session, msg, state):
# to a next batch. This module build is doomed.
if all([c.state != koji.BUILD_STATES['COMPLETE'] for c in current_batch]):
# They didn't all succeed.. so mark this module build as a failure.
parent.transition(config, rida.BUILD_STATES['failed'])
parent.transition(config, models.BUILD_STATES['failed'])
session.commit()
return

9
rida/scheduler/handlers/modules.py
View File

@@ -23,8 +23,8 @@
""" Handlers for module change events on the message bus. """
from rida import models, db, log
import rida.builder
import rida.database
import rida.pdc
import rida.utils
@@ -34,15 +34,16 @@ import logging
import os
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
def get_rpm_release_from_tag(tag):
return tag.replace("-", "_")
def get_artifact_from_srpm(srpm_path):
return os.path.basename(srpm_path).replace(".src.rpm", "")
def wait(config, session, msg):
""" Called whenever a module enters the 'wait' state.
@@ -52,7 +53,7 @@ def wait(config, session, msg):
The kicking off of individual component builds is handled elsewhere,
in rida.schedulers.handlers.repos.
"""
build = rida.database.ModuleBuild.from_module_event(session, msg)
build = models.ModuleBuild.from_module_event(db.session, msg)
log.info("Found build=%r from message" % build)
module_info = build.json()
@@ -111,7 +112,7 @@ def wait(config, session, msg):
artifact_name = "module-build-macros"
task_id = builder.build(artifact_name=artifact_name, source=srpm)
component_build = rida.database.ComponentBuild(
component_build = models.ComponentBuild(
module_id=build.id,
package=artifact_name,
format="rpms",

11
rida/scheduler/handlers/repos.py
View File

@@ -24,13 +24,12 @@
""" Handlers for repo change events on the message bus. """
import rida.builder
import rida.database
import rida.pdc
import logging
import koji
from rida import models, log
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
def done(config, session, msg):
@@ -38,7 +37,7 @@ def done(config, session, msg):
# First, find our ModuleBuild associated with this repo, if any.
tag = msg['msg']['tag'].strip('-build')
module_build = rida.database.ModuleBuild.from_repo_done_event(session, msg)
module_build = models.ModuleBuild.from_repo_done_event(session, msg)
if not module_build:
log.info("No module build found associated with koji tag %r" % tag)
return
@@ -46,7 +45,7 @@ def done(config, session, msg):
# It is possible that we have already failed.. but our repo is just being
# routinely regenerated. Just ignore that. If rida says the module is
# dead, then the module is dead.
if module_build.state == rida.BUILD_STATES['failed']:
if module_build.state == models.BUILD_STATES['failed']:
log.info("Ignoring repo regen for already failed %r" % module_build)
return
@@ -71,7 +70,7 @@ def done(config, session, msg):
# first before we ever get here. This is here as a race condition safety
# valve.
if not good:
module_build.transition(config, rida.BUILD_STATES['failed'])
module_build.transition(config, models.BUILD_STATES['failed'])
session.commit()
log.warn("Odd! All components in batch failed for %r." % module_build)
return
@@ -103,7 +102,7 @@ def done(config, session, msg):
rida.utils.start_next_build_batch(
module_build, session, builder, components=leftover_components)
else:
module_build.transition(config, state=rida.BUILD_STATES['done'])
module_build.transition(config, state=models.BUILD_STATES['done'])
session.commit()
# And that's it. :)

82
rida/scheduler/main.py
View File

@@ -31,7 +31,6 @@ proper scheduling component builds in the supported build systems.
import inspect
import logging
import operator
import os
import pprint
@@ -47,26 +46,14 @@ except ImportError:
import rida.config
import rida.logger
import rida.messaging
import rida.scheduler.handlers.components
import rida.scheduler.handlers.modules
import rida.scheduler.handlers.repos
import sys
import koji
log = logging.getLogger(__name__)
# Load config from git checkout or the default location
config = None
here = sys.path[0]
if here not in ('/usr/bin', '/bin', '/usr/local/bin'):
# git checkout
config = rida.config.from_file("rida.conf")
else:
# production
config = rida.config.from_file()
from rida import conf, db, models, log
class STOP_WORK(object):
@@ -77,7 +64,7 @@ class STOP_WORK(object):
def module_build_state_from_msg(msg):
state = int(msg['msg']['state'])
# TODO better handling
assert state in rida.BUILD_STATES.values(), "state=%s(%s) is not in %s" % (state, type(state), rida.BUILD_STATES.values())
assert state in models.BUILD_STATES.values(), "state=%s(%s) is not in %s" % (state, type(state), models.BUILD_STATES.values())
return state
@@ -88,7 +75,7 @@ class MessageIngest(threading.Thread):
def run(self):
for msg in rida.messaging.listen(backend=config.messaging):
for msg in rida.messaging.listen(backend=conf.messaging):
self.outgoing_work_queue.put(msg)
@@ -109,12 +96,12 @@ class MessageWorker(threading.Thread):
koji.BUILD_STATES["DELETED"]: NO_OP,
}
self.on_module_change = {
rida.BUILD_STATES["init"]: NO_OP,
rida.BUILD_STATES["wait"]: rida.scheduler.handlers.modules.wait,
rida.BUILD_STATES["build"]: NO_OP,
rida.BUILD_STATES["failed"]: NO_OP,
rida.BUILD_STATES["done"]: NO_OP,
rida.BUILD_STATES["ready"]: NO_OP,
models.BUILD_STATES["init"]: NO_OP,
models.BUILD_STATES["wait"]: rida.scheduler.handlers.modules.wait,
models.BUILD_STATES["build"]: NO_OP,
models.BUILD_STATES["failed"]: NO_OP,
models.BUILD_STATES["done"]: NO_OP,
models.BUILD_STATES["ready"]: NO_OP,
}
# Only one kind of repo change event, though...
self.on_repo_change = rida.scheduler.handlers.repos.done
@@ -122,8 +109,8 @@ class MessageWorker(threading.Thread):
def sanity_check(self):
""" On startup, make sure our implementation is sane. """
# Ensure we have every state covered
for state in rida.BUILD_STATES:
if rida.BUILD_STATES[state] not in self.on_module_change:
for state in models.BUILD_STATES:
if models.BUILD_STATES[state] not in self.on_module_change:
raise KeyError("Module build states %r not handled." % state)
for state in koji.BUILD_STATES:
if koji.BUILD_STATES[state] not in self.on_build_change:
@@ -176,10 +163,9 @@ class MessageWorker(threading.Thread):
if handler is self.NO_OP:
log.debug("Handler is NO_OP: %s" % idx)
else:
with rida.database.Database(config) as session:
log.info("Calling %s" % idx)
handler(config, session, msg)
log.info("Done with %s" % idx)
log.info("Calling %s" % idx)
handler(conf, db.session, msg)
log.info("Done with %s" % idx)
class Poller(threading.Thread):
@@ -189,20 +175,15 @@ class Poller(threading.Thread):
def run(self):
while True:
with rida.database.Database(config) as session:
self.log_summary(session)
self.log_summary(db.session)
# XXX: detect whether it's really stucked first
#with rida.database.Database(config) as session:
# self.process_waiting_module_builds(session)
with rida.database.Database(config) as session:
self.process_open_component_builds(session)
with rida.database.Database(config) as session:
self.process_lingering_module_builds(session)
with rida.database.Database(config) as session:
self.fail_lost_builds(session)
# self.process_waiting_module_builds(db.session)
self.process_open_component_builds(db.session)
self.process_lingering_module_builds(db.session)
self.fail_lost_builds(db.session)
log.info("Polling thread sleeping, %rs" % config.polling_interval)
time.sleep(config.polling_interval)
log.info("Polling thread sleeping, %rs" % conf.polling_interval)
time.sleep(conf.polling_interval)
def fail_lost_builds(self, session):
# This function is supposed to be handling only
@@ -211,12 +192,11 @@ class Poller(threading.Thread):
# TODO re-use
if config.system == "koji":
koji_session, _ = rida.builder.KojiModuleBuilder.get_session_from_config(config)
if conf.system == "koji":
koji_session, _ = rida.builder.KojiModuleBuilder.get_session_from_config(conf)
state = koji.BUILD_STATES['BUILDING'] # Check tasks that we track as BUILDING
log.info("Querying tasks for statuses:")
query = session.query(rida.database.ComponentBuild)
res = query.filter(state==koji.BUILD_STATES['BUILDING']).all()
res = models.ComponentBuild.query.filter_by(state=koji.BUILD_STATES['BUILDING']).all()
log.info("Checking status for %d tasks." % len(res))
for component_build in res:
@@ -245,16 +225,16 @@ class Poller(threading.Thread):
})
else:
raise NotImplementedError("Buildsystem %r is not supported." % config.system)
raise NotImplementedError("Buildsystem %r is not supported." % conf.system)
def log_summary(self, session):
log.info("Current status:")
backlog = self.outgoing_work_queue.qsize()
log.info(" * internal queue backlog is %i." % backlog)
states = sorted(rida.BUILD_STATES.items(), key=operator.itemgetter(1))
states = sorted(models.BUILD_STATES.items(), key=operator.itemgetter(1))
for name, code in states:
query = session.query(rida.database.ModuleBuild)
count = query.filter_by(state=code).count()
query = models.ModuleBuild.query.filter_by(state=code)
count = query.count()
if count:
log.info(" * %i module builds in the %s state." % (count, name))
if name == 'build':
@@ -263,10 +243,9 @@ class Poller(threading.Thread):
for component_build in module_build.component_builds:
log.info(" * %r" % component_build)
def process_waiting_module_builds(self, session):
log.info("Looking for module builds stuck in the wait state.")
builds = rida.database.ModuleBuild.by_state(session, "wait")
builds = models.ModuleBuild.by_state(session, "wait")
# TODO -- do throttling calculation here...
log.info(" %r module builds in the wait state..." % len(builds))
for build in builds:
@@ -275,7 +254,7 @@ class Poller(threading.Thread):
'topic': '.module.build.state.change',
'msg': build.json(),
}
rida.scheduler.handlers.modules.wait(config, session, msg)
rida.scheduler.handlers.modules.wait(conf, session, msg)
def process_open_component_builds(self, session):
log.warning("process_open_component_builds is not yet implemented...")
@@ -285,7 +264,6 @@ class Poller(threading.Thread):
def main():
rida.logger.init_logging(config)
log.info("Starting ridad.")
try:
work_queue = queue.Queue()

4
rida/scm.py
View File

@@ -34,9 +34,7 @@ import subprocess as sp
import re
import tempfile
import logging
log = logging.getLogger(__name__)
from rida import log
import rida.utils

4
rida/utils.py
View File

@@ -22,10 +22,8 @@
""" Utility functions for rida. """
import functools
import logging
import time
log = logging.getLogger(__name__)
from rida import log
def retry(timeout=120, interval=30, wait_on=Exception):

84
rida.py → rida/views.py Executable file → Normal file
View File

@@ -22,39 +22,23 @@
#
# Written by Petr Šabata <contyk@redhat.com>
"""The module build orchestrator for Modularity, API.
""" The module build orchestrator for Modularity, API.
This is the implementation of the orchestrator's public RESTful API.
"""
from flask import Flask, request
import flask
from flask import request, jsonify
import json
import logging
import modulemd
import os
import rida.auth
import rida.config
import rida.database
import rida.logger
import rida.scm
import ssl
import shutil
import tempfile
from rida import app, conf, db, log
from rida import models
app = Flask(__name__)
app.config.from_envvar("RIDA_SETTINGS", silent=True)
ridaconfig=os.environ.get('RIDA_CONFIG')
if ridaconfig:
conf = rida.config.from_file(ridaconfig)
else:
conf = rida.config.from_file("rida.conf")
rida.logger.init_logging(conf)
log = logging.getLogger(__name__)
db = rida.database.Database(conf)
@app.route("/rida/module-builds/", methods=["POST"])
def submit_build():
@@ -102,10 +86,10 @@ def submit_build():
mmd.loads(yaml)
except:
return "Invalid modulemd", 422
if db.session.query(rida.database.ModuleBuild).filter_by(name=mmd.name,
version=mmd.version, release=mmd.release).first():
if models.ModuleBuild.query.filter_by(name=mmd.name, version=mmd.version, release=mmd.release).first():
return "Module already exists", 409
module = rida.database.ModuleBuild.create(
module = models.ModuleBuild.create(
db.session,
conf,
name=mmd.name,
@@ -118,7 +102,7 @@ def submit_build():
def failure(message, code):
# TODO, we should make some note of why it failed in the db..
log.exception(message)
module.transition(conf, rida.database.BUILD_STATES["failed"])
module.transition(conf, models.BUILD_STATES["failed"])
db.session.add(module)
db.session.commit()
return message, code
@@ -140,7 +124,7 @@ def submit_build():
full_url = pkg["repository"] + "?#" + pkg["commit"]
if not rida.scm.SCM(full_url).is_available():
return failure("Cannot checkout %s" % pkgname, 422)
build = rida.database.ComponentBuild(
build = models.ComponentBuild(
module_id=module.id,
package=pkgname,
format="rpms",
@@ -148,70 +132,34 @@ def submit_build():
)
db.session.add(build)
module.modulemd = mmd.dumps()
module.transition(conf, rida.database.BUILD_STATES["wait"])
module.transition(conf, models.BUILD_STATES["wait"])
db.session.add(module)
db.session.commit()
logging.info("%s submitted build of %s-%s-%s", username, mmd.name,
mmd.version, mmd.release)
return flask.jsonify(module.json()), 201
return jsonify(module.json()), 201
@app.route("/rida/module-builds/", methods=["GET"])
def query_builds():
"""Lists all tracked module builds."""
return flask.jsonify([{"id": x.id, "state": x.state}
for x in db.session.query(rida.database.ModuleBuild).all()]), 200
return jsonify([{"id": x.id, "state": x.state}
for x in models.ModuleBuild.query.all()]), 200
@app.route("/rida/module-builds/<int:id>", methods=["GET"])
def query_build(id):
"""Lists details for the specified module builds."""
module = db.session.query(rida.database.ModuleBuild).filter_by(id=id).first()
module = models.ModuleBuild.query.filter_by(id=id).first()
if module:
tasks = dict()
if module.state != "init":
for build in db.session.query(rida.database.ComponentBuild).filter_by(module_id=id).all():
for build in models.ComponentBuild.query.filter_by(module_id=id).all():
tasks[build.format + "/" + build.package] = \
str(build.task_id) + "/" + build.state
return flask.jsonify({
return jsonify({
"id": module.id,
"state": module.state,
"tasks": tasks
}), 200
else:
return "No such module found.", 404
def _establish_ssl_context(conf):
if conf.ssl_enabled == False:
return None
# First, do some validation of the configuration
attributes = (
'ssl_certificate_file',
'ssl_certificate_key_file',
'ssl_ca_certificate_file',
)
for attribute in attributes:
value = getattr(conf, attribute, None)
if not value:
raise ValueError("%r could not be found" % attribute)
if not os.path.exists(value):
raise OSError("%s: %s file not found." % (attribute, value))
# Then, establish the ssl context and return it
ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_ctx.load_cert_chain(conf.ssl_certificate_file,
conf.ssl_certificate_key_file)
ssl_ctx.verify_mode = ssl.CERT_OPTIONAL
ssl_ctx.load_verify_locations(cafile=conf.ssl_ca_certificate_file)
return ssl_ctx
if __name__ == "__main__":
logging.info("Starting Rida")
ssl_ctx = _establish_ssl_context(conf)
app.run(
host=conf.host,
port=conf.port,
request_handler=rida.auth.ClientCertRequestHandler,
ssl_context=ssl_ctx,
)

28
test-buildroot.py
View File

@@ -1,28 +0,0 @@
#!/usr/bin/env python3
""" A little script to test buildroot creation. """
from rida.builder import KojiModuleBuild, Builder
from rida.config import Config
cfg = Config()
cfg.koji_profile = "koji"
cfg.koji_config = "/etc/rida/koji.conf"
cfg.koji_arches = ["x86_64", "i686"]
mb = KojiModuleBuild(module="testmodule-1.0", config=cfg) # or By using Builder
#mb = Builder(module="testmodule-1.0", backend="koji", config=cfg)
resume = False
if not resume:
mb.buildroot_prep()
mb.buildroot_add_dependency(["f24"])
mb.buildroot_ready()
task_id = mb.build(artifact_name="fedora-release", source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9")
mb.buildroot_add_artifacts(["fedora-release-24-2", ]) # just example with disttag macro
mb.buildroot_ready(artifact="fedora-release-24-2")
else:
mb.buildroot_resume()
task_id = mb.build(artifact_name="fedora-release", source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9")

22
test-pdc.py
View File

@@ -1,22 +0,0 @@
#!/usr/bin/env python3
""" A little script to test pdc interaction. """
from rida.pdc import *
from rida.config import Config
cfg = Config()
cfg.pdc_url = "http://modularity.fedorainfracloud.org:8080/rest_api/v1"
cfg.pdc_insecure = True
cfg.pdc_develop = True
pdc_session = get_pdc_client_session(cfg)
module = get_module(pdc_session, {'name': 'testmodule', 'version': '4.3.43', 'release': '1'})
if module:
print ("pdc_data=%s" % str(module))
print ("deps=%s" % get_module_runtime_dependencies(pdc_session, module))
print ("build_deps=%s" % get_module_build_dependencies(pdc_session, module))
print ("tag=%s" % get_module_tag(pdc_session, module))
else:
print ('module was not found')

2
tests/test_scheduler/test_module_wait.py
View File

@@ -34,7 +34,7 @@ class TestModuleWait(unittest.TestCase):
self.fn = rida.scheduler.handlers.modules.wait
@mock.patch('rida.builder.KojiModuleBuilder')
@mock.patch('rida.database.ModuleBuild.from_module_event')
@mock.patch('rida.models.ModuleBuild.from_module_event')
@mock.patch('rida.pdc')
def test_init_basic(self, pdc, from_module_event, KojiModuleBuilder):
builder = mock.Mock()

4
tests/test_scheduler/test_repo_done.py
View File

@@ -37,7 +37,7 @@ class TestRepoDone(unittest.TestCase):
self.session = mock.Mock()
self.fn = rida.scheduler.handlers.repos.done
@mock.patch('rida.database.ModuleBuild.from_repo_done_event')
@mock.patch('rida.models.ModuleBuild.from_repo_done_event')
def test_no_match(self, from_repo_done_event):
""" Test that when a repo msg hits us and we have no match,
that we do nothing gracefully.
@@ -52,7 +52,7 @@ class TestRepoDone(unittest.TestCase):
@mock.patch('rida.builder.KojiModuleBuilder.get_session_from_config')
@mock.patch('rida.builder.KojiModuleBuilder.build')
@mock.patch('rida.builder.KojiModuleBuilder.buildroot_resume')
@mock.patch('rida.database.ModuleBuild.from_repo_done_event')
@mock.patch('rida.models.ModuleBuild.from_repo_done_event')
def test_a_single_match(self, from_repo_done_event, resume, build_fn, config):
""" Test that when a repo msg hits us and we have no match,
that we do nothing gracefully.