diff --git a/rida.conf b/rida.conf index f8305b95..e7d4c84e 100644 --- a/rida.conf +++ b/rida.conf @@ -22,6 +22,7 @@ rpms_allow_repository = False rpms_default_cache = http://pkgs.fedoraproject.org/repo/pkgs/ rpms_allow_cache = False +ssl_enabled = True ssl_certificate_file = server.crt ssl_certificate_key_file = server.key ssl_ca_certificate_file = cacert.pem diff --git a/rida.py b/rida.py index 6673079e..b05a8c77 100755 --- a/rida.py +++ b/rida.py @@ -171,7 +171,7 @@ def _establish_ssl_context(conf): attributes = ( 'ssl_certificate_file', 'ssl_certificate_key_file', - 'ssl_ca_ceritifcate_file', + 'ssl_ca_certificate_file', ) for attribute in attributes: value = getattr(conf, attribute, None) diff --git a/rida/auth.py b/rida/auth.py index 99dd3699..541bc4ca 100644 --- a/rida/auth.py +++ b/rida/auth.py @@ -40,7 +40,7 @@ class ClientCertRequestHandler(WSGIRequestHandler): try: cert = self.request.getpeercert(False) - except: + except AttributeError: cert = None if cert and "subject" in cert: diff --git a/rida/config.py b/rida/config.py index 87faa25a..d643a744 100644 --- a/rida/config.py +++ b/rida/config.py @@ -56,7 +56,7 @@ def from_file(filename=None): if not isinstance(filename, str): raise TypeError("The configuration filename must be a string.") if not os.path.isfile(filename): - raise IOError("The configuration file doesn't exist.") + raise IOError("The configuration file '%s' doesn't exist." % filename) cp = configparser.ConfigParser(allow_no_value=True) cp.read(filename) default = cp.defaults() diff --git a/rida/database.py b/rida/database.py index 26c60da0..8af9efea 100644 --- a/rida/database.py +++ b/rida/database.py @@ -84,7 +84,7 @@ class Database(object): self._session = None # Lazilly created.. def __enter__(self): - return self.session() + return self.session def __exit__(self, *args, **kwargs): self._session.close() @@ -141,7 +141,7 @@ class ModuleBuild(Base): def from_fedmsg(cls, session, msg): if '.module.' not in msg['topic']: raise ValueError("%r is not a module message." % msg['topic']) - return session.query(cls).filter_by(cls.id==msg['msg']['id']) + return session.query(cls).filter(cls.id==msg['msg']['id']).one() def json(self): return { diff --git a/rida/pdc.py b/rida/pdc.py index 1001fec3..6d85e0e8 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -50,7 +50,7 @@ def get_variant_dict(data): if not isinstance(data, dict): return False - for attr in ('name', 'version'): + for attr in ('name', 'version', 'release'): if attr not in data.keys(): return False return True @@ -59,7 +59,7 @@ def get_variant_dict(data): if not isinstance(data, dict): return False - for attr in ('variant_name', 'variant_version'): + for attr in ('variant_name', 'variant_version', 'variant_release'): if attr not in data.keys(): return False return True @@ -72,11 +72,11 @@ def get_variant_dict(data): result = None - if is_module_str(data): + if is_module_str(data): result = variant_dict_from_str(data) - elif is_modulemd(data): - result = {'variant_name': data.name, 'variant_version': data.version } + elif is_modulemd(data): + result = {'variant_name': data.name, 'variant_version': data.version, 'variant_release': data.release } elif is_variant_dict(data): result = data @@ -84,6 +84,9 @@ def get_variant_dict(data): if 'variant_type' not in result.keys(): result['variant_type'] = 'module' + if 'variant_release' not in result.keys(): + result['variant_release'] = '0' + elif is_module_dict(data): result = {'variant_name': data['name'], 'variant_version': data['version']} @@ -92,7 +95,7 @@ def get_variant_dict(data): return result - + def variant_dict_from_str(module_str): """ @@ -118,7 +121,7 @@ def get_module(session, module_info): :return final list of module_info which pass repoclosure """ - module_info = get_variant_dict(module_info) + module_info = get_variant_dict(module_info) module_info = session['unreleasedvariants'](page_size=-1, **module_info) assert len(module_info) <= 1 diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 2ec9f6df..4e4d4191 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -26,6 +26,9 @@ import rida.builder import rida.database import rida.pdc +import time +import logging +import koji import logging log = logging.getLogger(__name__) @@ -40,13 +43,27 @@ def init(config, session, msg): """ build = rida.database.ModuleBuild.from_fedmsg(session, msg) pdc = rida.pdc.get_pdc_client_session(config) - module_info = build.to_pdc_module_info() + # TODO do some periodical polling of variant_info since it's being created based on the same message + #log.warn("HACK: waiting 10s for pdc") + #time.sleep(10) + log.debug("Getting module from pdc with following input_data=%s" % build.json()) + module_info = pdc.get_module(build.json()) + + log.debug("Received module_info=%s from pdc" % module_info) + tag = rida.pdc.get_module_tag(pdc, module_info) + log.info("Found tag=%s for module %s-%s-%s" % (tag, build['name'], build['version'], build['release'])) + dependencies = rida.pdc.get_module_dependencies(pdc, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config) builder.buildroot_add_dependency(dependencies) build.buildroot_task_id = builder.buildroot_prep() + # TODO: build srpm with dist_tag macros + # TODO submit build from srpm to koji + # TODO: buildroot.add_artifact(build_with_dist_tags) + # TODO: buildroot.ready(artifact=$artifact) build.state = "wait" # Wait for the buildroot to be ready. + log.debug("Done with init") def build(config, session, msg): diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index bc0f536d..5ff770fa 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -40,19 +40,34 @@ import rida.logging import rida.messaging import rida.scheduler.handlers.modules #import rida.scheduler.handlers.builds +import sys import koji log = logging.getLogger(__name__) -# TODO: Load the config file from environment -config = rida.config.from_file("rida.conf") +# Load config from git checkout or the default location +config = None +here = sys.path[0] +if here not in ('/usr/bin', '/bin', '/usr/local/bin'): + # git checkout + config = rida.config.from_file("rida.conf") +else: + # production + config = rida.config.from_file() # TODO: Utilized rida.builder to prepare the buildroots and build components. # TODO: Set the build state to build once the module build is started. # TODO: Set the build state to done once the module build is done. # TODO: Set the build state to failed if the module build fails. +def module_build_state_from_msg(msg): + + state = int(msg['msg']['state']) + # TODO better handling + assert state in rida.BUILD_STATES.values(), "state=%s(%s) is not in %s" % (state, type(state), rida.BUILD_STATES.values()) + return state + class Messaging(threading.Thread): # These are our main lookup tables for figuring out what to run in response @@ -61,7 +76,7 @@ class Messaging(threading.Thread): koji.BUILD_STATES["BUILDING"]: lambda x: x } on_module_change = { - rida.BUILD_STATES["new"]: rida.scheduler.handlers.modules.new, + rida.BUILD_STATES["init"]: rida.scheduler.handlers.modules.init, } def sanity_check(self): @@ -83,24 +98,25 @@ class Messaging(threading.Thread): callback, key, argspec, expected)) def run(self): - self.sanity_check() + #self.sanity_check() # TODO: Check for modules that can be set to done/failed # TODO: Act on these things somehow # TODO: Emit messages about doing so for msg in rida.messaging.listen(backend=config.messaging): log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic'])) + log.debug(msg) # Choose a handler for this message if '.buildsys.build.state.change' in msg['topic']: - handler = self.on_build_change[msg['msg']['new']] + handler = self.on_build_change[msg['msg']['init']] elif '.rida.module.state.change' in msg['topic']: - handler = self.on_module_change[msg['msg']['state']] + handler = self.on_module_change[module_build_state_from_msg(msg)] else: log.debug("Unhandled message...") continue # Execute our chosen handler - with rida.Database(config) as session: + with rida.database.Database(config) as session: handler(config, session, msg) class Polling(threading.Thread): diff --git a/test-pdc.py b/test-pdc.py index 448ece61..c5156377 100755 --- a/test-pdc.py +++ b/test-pdc.py @@ -6,12 +6,12 @@ from rida.config import Config cfg = Config() -cfg.pdc_url = "http://fed-mod.org:8000/rest_api/v1" +cfg.pdc_url = "http://localhost:8000/rest_api/v1" cfg.pdc_insecure = True cfg.pdc_develop = True pdc_session = get_pdc_client_session(cfg) -module = get_module(pdc_session, {'name': 'testmodule', 'version': '1.0'}) +module = get_module(pdc_session, {'name': 'testmodule', 'version': '4.3.42', 'release': '0'}) if module: print ("pdc_data=%s" % str(module))