From 64ce1a517a9bf18585a7d31b8c152470a8cd934b Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Thu, 14 Jul 2016 14:32:04 +0200 Subject: [PATCH 01/13] fixed typo in rida.py --- rida.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida.py b/rida.py index 61995ac5..58cc415d 100755 --- a/rida.py +++ b/rida.py @@ -171,7 +171,7 @@ def _establish_ssl_context(conf): attributes = ( 'ssl_certificate_file', 'ssl_certificate_key_file', - 'ssl_ca_ceritifcate_file', + 'ssl_ca_certificate_file', ) for attribute in attributes: value = getattr(conf, attribute, None) From 70ca8ee88511db40e64d1483ea30a9bd31548e67 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Thu, 14 Jul 2016 14:32:38 +0200 Subject: [PATCH 02/13] except -> except AttributeError for getpeercert --- rida/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/auth.py b/rida/auth.py index 99dd3699..541bc4ca 100644 --- a/rida/auth.py +++ b/rida/auth.py @@ -40,7 +40,7 @@ class ClientCertRequestHandler(WSGIRequestHandler): try: cert = self.request.getpeercert(False) - except: + except AttributeError: cert = None if cert and "subject" in cert: From 490da57176a068836321dcb01435677c22914cff Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 15:05:10 +0200 Subject: [PATCH 03/13] config: add ssl_enabled=True. fas auth would simply not work without ssl --- rida.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/rida.conf b/rida.conf index f8305b95..e7d4c84e 100644 --- a/rida.conf +++ b/rida.conf @@ -22,6 +22,7 @@ rpms_allow_repository = False rpms_default_cache = http://pkgs.fedoraproject.org/repo/pkgs/ rpms_allow_cache = False +ssl_enabled = True ssl_certificate_file = server.crt ssl_certificate_key_file = server.key ssl_ca_certificate_file = cacert.pem From ef48854aa67f750d561f744bf668a9902dfc2ee2 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 15:05:35 +0200 Subject: [PATCH 04/13] IOError prints filename. Very useful for debugging --- rida/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/config.py b/rida/config.py index 87faa25a..d643a744 100644 --- a/rida/config.py +++ b/rida/config.py @@ -56,7 +56,7 @@ def from_file(filename=None): if not isinstance(filename, str): raise TypeError("The configuration filename must be a string.") if not os.path.isfile(filename): - raise IOError("The configuration file doesn't exist.") + raise IOError("The configuration file '%s' doesn't exist." % filename) cp = configparser.ConfigParser(allow_no_value=True) cp.read(filename) default = cp.defaults() From d53f57f23044dd4640781925f62cdc3423cdb90c Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 15:06:32 +0200 Subject: [PATCH 05/13] database module: fixed session is not callable and fixed filter_by -> filter (previously raised error) --- rida/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/database.py b/rida/database.py index 62706968..2781379e 100644 --- a/rida/database.py +++ b/rida/database.py @@ -68,7 +68,7 @@ class Database(object): self._session = None # Lazilly created.. def __enter__(self): - return self.session() + return self.session def __exit__(self, *args, **kwargs): self._session.close() @@ -125,7 +125,7 @@ class ModuleBuild(Base): def from_fedmsg(cls, session, msg): if '.module.' not in msg['topic']: raise ValueError("%r is not a module message." % msg['topic']) - return session.query(cls).filter_by(cls.id==msg['msg']['id']) + return session.query(cls).filter(cls.id==msg['msg']['id']) def json(self): return { From 4eae650878d116a68a52337dd166abcbbefdeb0d Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 15:06:53 +0200 Subject: [PATCH 06/13] rida/pdc.py: add release --- rida/pdc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rida/pdc.py b/rida/pdc.py index 0d7bddbe..bba0b1e7 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -50,7 +50,7 @@ def get_variant_dict(data): if not isinstance(data, dict): return False - for attr in ('name', 'version'): + for attr in ('name', 'version', 'release'): if attr not in data.keys(): return False return True @@ -59,7 +59,7 @@ def get_variant_dict(data): if not isinstance(data, dict): return False - for attr in ('variant_name', 'variant_version'): + for attr in ('variant_name', 'variant_version', 'variant_release'): if attr not in data.keys(): return False return True @@ -76,7 +76,7 @@ def get_variant_dict(data): result = variant_dict_from_str(data) elif is_modulemd(data): - result = {'variant_name': data.name, 'variant_version': data.version } + result = {'variant_name': data.name, 'variant_version': data.version, 'variant_release': data.release } elif is_variant_dict(data): result = data @@ -84,6 +84,9 @@ def get_variant_dict(data): if 'variant_type' not in result.keys(): result['variant_type'] = 'module' + if 'variant_release' not in result.keys(): + result['variant_release'] = '0' + elif is_module_dict(data): result = {'variant_name': data['name'], 'variant_version': data['version']} From 5222b25c97f77234355c86f36132aed98e7f311f Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 15:09:36 +0200 Subject: [PATCH 07/13] Fixed new -> init, temp. disabled sanity_check, handling states correctly Signed-off-by: Lubos Kocman --- rida/scheduler/main.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index bc7cf87c..dbbedd74 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -39,19 +39,34 @@ import rida.config import rida.messaging import rida.scheduler.handlers.modules #import rida.scheduler.handlers.builds +import sys import koji log = logging.getLogger() -# TODO: Load the config file from environment -config = rida.config.from_file("rida.conf") +# Load config from git checkout or the default location +config = None +here = sys.path[0] +if here not in ('/usr/bin', '/bin', '/usr/local/bin'): + # git checkout + config = rida.config.from_file("rida.conf") +else: + # production + config = rida.config.from_file() # TODO: Utilized rida.builder to prepare the buildroots and build components. # TODO: Set the build state to build once the module build is started. # TODO: Set the build state to done once the module build is done. # TODO: Set the build state to failed if the module build fails. +def module_build_state_from_msg(msg): + + state = int(msg['msg']['state']) + # TODO better handling + assert state in rida.BUILD_STATES.values(), "state=%s(%s) is not in %s" % (state, type(state), rida.BUILD_STATES.values()) + return state + class Messaging(threading.Thread): # These are our main lookup tables for figuring out what to run in response @@ -60,7 +75,7 @@ class Messaging(threading.Thread): koji.BUILD_STATES["BUILDING"]: lambda x: x } on_module_change = { - rida.BUILD_STATES["new"]: rida.scheduler.handlers.modules.new, + rida.BUILD_STATES["init"]: rida.scheduler.handlers.modules.init, } def sanity_check(self): @@ -82,24 +97,25 @@ class Messaging(threading.Thread): callback, key, argspec, expected)) def run(self): - self.sanity_check() + #self.sanity_check() # TODO: Check for modules that can be set to done/failed # TODO: Act on these things somehow # TODO: Emit messages about doing so for msg in rida.messaging.listen(backend=config.messaging): log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic'])) + log.debug(msg) # Choose a handler for this message if '.buildsys.build.state.change' in msg['topic']: - handler = self.on_build_change[msg['msg']['new']] + handler = self.on_build_change[msg['msg']['init']] elif '.rida.module.state.change' in msg['topic']: - handler = self.on_module_change[msg['msg']['state']] + handler = self.on_module_change[module_build_state_from_msg(msg)] else: log.debug("Unhandled message...") continue # Execute our chosen handler - with rida.Database(config) as session: + with rida.database.Database(config) as session: handler(config, session, msg) class Polling(threading.Thread): From ec01f80189c07d554e514500cf4b0812218b9832 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 15:10:50 +0200 Subject: [PATCH 08/13] temp. sleepy-hack to wait for pdc-updater to finish his job --- rida/scheduler/handlers/modules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 8df6342e..0de95d37 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -26,6 +26,7 @@ import rida.builder import rida.database import rida.pdc +import time def init(config, session, msg): @@ -37,6 +38,8 @@ def init(config, session, msg): """ build = rida.database.ModuleBuild.from_fedmsg(session, msg) pdc = rida.pdc.get_pdc_client_session(config) + # TODO do some periodical polling of variant_info since it's being created based on the same message + time.sleep(10) module_info = build.to_pdc_module_info() tag = rida.pdc.get_module_tag(pdc, module_info) dependencies = rida.pdc.get_module_dependencies(pdc, module_info) From 1e44e3b2993cdfbaebe3ddcf7f1aa618e2445afc Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 16:04:33 +0200 Subject: [PATCH 09/13] pdc: unreleasedvariants --- rida/pdc.py | 10 +++++----- test-pdc.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rida/pdc.py b/rida/pdc.py index bba0b1e7..ba375aa2 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -72,10 +72,10 @@ def get_variant_dict(data): result = None - if is_module_str(data): + if is_module_str(data): result = variant_dict_from_str(data) - elif is_modulemd(data): + elif is_modulemd(data): result = {'variant_name': data.name, 'variant_version': data.version, 'variant_release': data.release } elif is_variant_dict(data): @@ -95,7 +95,7 @@ def get_variant_dict(data): return result - + def variant_dict_from_str(module_str): """ @@ -121,9 +121,9 @@ def get_module(session, module_info): :return final list of module_info which pass repoclosure """ - module_info = get_variant_dict(module_info) + module_info = get_variant_dict(module_info) - module_info = session['unreleasedvariant'](page_size=-1, **module_info) + module_info = session['unreleasedvariants'](page_size=-1, **module_info) assert len(module_info) <= 1 if not module_info: diff --git a/test-pdc.py b/test-pdc.py index 448ece61..c5156377 100755 --- a/test-pdc.py +++ b/test-pdc.py @@ -6,12 +6,12 @@ from rida.config import Config cfg = Config() -cfg.pdc_url = "http://fed-mod.org:8000/rest_api/v1" +cfg.pdc_url = "http://localhost:8000/rest_api/v1" cfg.pdc_insecure = True cfg.pdc_develop = True pdc_session = get_pdc_client_session(cfg) -module = get_module(pdc_session, {'name': 'testmodule', 'version': '1.0'}) +module = get_module(pdc_session, {'name': 'testmodule', 'version': '4.3.42', 'release': '0'}) if module: print ("pdc_data=%s" % str(module)) From 1857b109a2ca0440b8cff8f2b20f5415ad255d51 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 17:53:36 +0200 Subject: [PATCH 10/13] database: use .one() instead of .all() + assert Signed-off-by: Lubos Kocman --- rida/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index 2781379e..6163e8b6 100644 --- a/rida/database.py +++ b/rida/database.py @@ -125,7 +125,7 @@ class ModuleBuild(Base): def from_fedmsg(cls, session, msg): if '.module.' not in msg['topic']: raise ValueError("%r is not a module message." % msg['topic']) - return session.query(cls).filter(cls.id==msg['msg']['id']) + return session.query(cls).filter(cls.id==msg['msg']['id']).one() def json(self): return { From c6049da3015453a20819347669cc8d9270207587 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 17:55:51 +0200 Subject: [PATCH 11/13] modules.py: updated db to pdc calls --- rida/scheduler/handlers/modules.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 0de95d37..c5ef9aac 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -27,7 +27,11 @@ import rida.builder import rida.database import rida.pdc import time +import logging +import koji +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger() def init(config, session, msg): """ Called whenever a module enters the 'init' state. @@ -39,14 +43,22 @@ def init(config, session, msg): build = rida.database.ModuleBuild.from_fedmsg(session, msg) pdc = rida.pdc.get_pdc_client_session(config) # TODO do some periodical polling of variant_info since it's being created based on the same message - time.sleep(10) - module_info = build.to_pdc_module_info() + #log.warn("HACK: waiting 10s for pdc") + #time.sleep(10) + log.debug("Getting module from pdc with following input_data=%s" % build.json()) + module_info = pdc.get_module(build.json()) + + log.debug("Received module_info=%s from pdc" % module_info) + tag = rida.pdc.get_module_tag(pdc, module_info) + log.info("Found tag=%s for module %s-%s-%s" % (tag, build['name'], build['version'], build['release'])) + dependencies = rida.pdc.get_module_dependencies(pdc, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config) builder.buildroot_add_dependency(dependencies) build.buildroot_task_id = builder.buildroot_prep() build.state = "wait" # Wait for the buildroot to be ready. + log.debug("Done with init") def build(config, session, msg): From 8d17d3a7630663dac07dd87367be681cc290d7fe Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 15 Jul 2016 18:04:16 +0200 Subject: [PATCH 12/13] Add bunch of todos for init phase Signed-off-by: Lubos Kocman --- rida/scheduler/handlers/modules.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index c5ef9aac..6587b1e6 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -57,6 +57,10 @@ def init(config, session, msg): builder = rida.builder.KojiModuleBuilder(build.name, config) builder.buildroot_add_dependency(dependencies) build.buildroot_task_id = builder.buildroot_prep() + # TODO: build srpm with dist_tag macros + # TODO submit build from srpm to koji + # TODO: buildroot.add_artifact(build_with_dist_tags) + # TODO: buildroot.ready(artifact=$artifact) build.state = "wait" # Wait for the buildroot to be ready. log.debug("Done with init") From 2eee1c8ed66602958dc9462222858842c336ba1a Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 11:23:32 -0400 Subject: [PATCH 13/13] Descriptions of the states. --- README.rst | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ rida.py | 2 +- rida/database.py | 16 +++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 96ebb84d..6ceb3ed2 100644 --- a/README.rst +++ b/README.rst @@ -131,3 +131,54 @@ Possible response codes are for various requests include: - HTTP 501 Not Implemented - The requested URL is valid but the handler isn't implemented yet. - HTTP 503 Service Unavailable - The service is down, possibly for maintanance. + +Module Build States +------------------- + +You can see the list of possible states with:: + + import rida + print(rida.BUILD_STATES) + +Here's a description of what each of them means: + +init +~~~~ + +This is (obviously) the first state a module build enters. + +When a user first submits a module build, it enters this state. We parse the +modulemd file, learn the NVR, and create a record for the module build. + +Then, we validate that the components are available, and that we can fetch +them. If this is all good, then we set the build to the 'wait' state. If +anything goes wrong, we jump immediately to the 'failed' state. + +wait +~~~~ + +Here, the scheduler picks up tasks in wait and switches to build immediately. +Eventually, we'll add throttling logic here so we don't submit too many builds for the build system to handle. + +build +~~~~~ + +The scheduler works on builds in this state. We prepare the buildroot, submit +builds for all the components, and wait for the results to come back. + +done +~~~~ + +Once all components have succeeded, we set the top-level module build to 'done'. + +failed +~~~~~~ + +If any of the component builds fail, then we set the top-level module build to 'failed' also. + +ready +~~~~~ + +This is a state to be set when a module is ready to be part of a +larger compose. perhaps it is set by an external service that knows +about the Grand Plan. diff --git a/rida.py b/rida.py index 58cc415d..b05a8c77 100755 --- a/rida.py +++ b/rida.py @@ -123,7 +123,7 @@ def submit_build(): build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() - module.state = rida.database.BUILD_STATES["init"] + module.state = rida.database.BUILD_STATES["wait"] db.session.add(module) db.session.commit() # Publish to whatever bus we're configured to connect to. diff --git a/rida/database.py b/rida/database.py index 6163e8b6..8af9efea 100644 --- a/rida/database.py +++ b/rida/database.py @@ -41,11 +41,27 @@ from sqlalchemy.ext.declarative import declarative_base # Just like koji.BUILD_STATES, except our own codes for modules. BUILD_STATES = { + # When you parse the modulemd file and know the nvr and you create a + # record in the db, and that's it. + # publish the message + # validate that components are available + # and that you can fetch them. + # if all is good, go to wait: telling ridad to take over. + # if something is bad, go straight to failed. "init": 0, + # Here, the scheduler picks up tasks in wait. + # switch to build immediately. + # throttling logic (when we write it) goes here. "wait": 1, + # Actively working on it. "build": 2, + # All is good "done": 3, + # Something failed "failed": 4, + # This is a state to be set when a module is ready to be part of a + # larger compose. perhaps it is set by an external service that knows + # about the Grand Plan. "ready": 5, }