From 64abebfd8993bd4ef0725e8d030aefda79d6169e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 12:39:34 -0400 Subject: [PATCH 001/106] Remove unused imports. --- rida/scheduler/handlers/modules.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 4e4d4191..5f7f1070 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -26,11 +26,9 @@ import rida.builder import rida.database import rida.pdc -import time import logging import koji -import logging log = logging.getLogger(__name__) From 434dc8bad00cd89cbdc333106d59d1b0c9fe0197 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 12:39:47 -0400 Subject: [PATCH 002/106] Simplify and clean. --- rida/scheduler/handlers/modules.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 5f7f1070..c4290fd4 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -41,11 +41,10 @@ def init(config, session, msg): """ build = rida.database.ModuleBuild.from_fedmsg(session, msg) pdc = rida.pdc.get_pdc_client_session(config) - # TODO do some periodical polling of variant_info since it's being created based on the same message - #log.warn("HACK: waiting 10s for pdc") - #time.sleep(10) - log.debug("Getting module from pdc with following input_data=%s" % build.json()) - module_info = pdc.get_module(build.json()) + + build_data = build.json() + log.debug("Getting module from pdc with input_data=%s" % build_data) + module_info = pdc.get_module(build_data) log.debug("Received module_info=%s from pdc" % module_info) From b60dc3231272fb4cda015a029027ed453423cdb3 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 13:00:04 -0400 Subject: [PATCH 003/106] Resolve ambiguity about what "pdc" refers to here... --- rida/scheduler/handlers/modules.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index c4290fd4..de25b72f 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -40,18 +40,18 @@ def init(config, session, msg): All we do here is request preparation of the buildroot. """ build = rida.database.ModuleBuild.from_fedmsg(session, msg) - pdc = rida.pdc.get_pdc_client_session(config) + pdc_session = rida.pdc.get_pdc_client_session(config) build_data = build.json() log.debug("Getting module from pdc with input_data=%s" % build_data) - module_info = pdc.get_module(build_data) + module_info = rida.pdc.get_module(pdc_session, build_data) log.debug("Received module_info=%s from pdc" % module_info) - tag = rida.pdc.get_module_tag(pdc, module_info) + tag = rida.pdc.get_module_tag(pdc_session, module_info) log.info("Found tag=%s for module %s-%s-%s" % (tag, build['name'], build['version'], build['release'])) - dependencies = rida.pdc.get_module_dependencies(pdc, module_info) + dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config) builder.buildroot_add_dependency(dependencies) build.buildroot_task_id = builder.buildroot_prep() From d94042f8e78c288ebe37f51a020d462fd36983f2 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 13:06:05 -0400 Subject: [PATCH 004/106] Add reprs for some of our models. --- rida/database.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index 8af9efea..8504bad7 100644 --- a/rida/database.py +++ b/rida/database.py @@ -158,6 +158,10 @@ class ModuleBuild(Base): 'component_builds': [build.id for build in self.component_builds], } + def __repr__(self): + return "" % ( + self.name, self.version, self.release) + class ComponentBuild(Base): __tablename__ = "component_builds" @@ -179,5 +183,8 @@ class ComponentBuild(Base): 'format': self.format, 'task': self.task, 'state': self.state, - 'module_build': self.module_build.id, + 'module_build': self.module_id, } + + def __repr__(self): + return "" % (self.package, self.module_id) From 9e3efc3fb47612c0904ba126d0fc9fa03c947808 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 13:06:26 -0400 Subject: [PATCH 005/106] Remove a bunch of unnecessary stuff. This is because get_module() is already called *inside* the call to get_module_tag() and get_module_dependencies(). --- rida/pdc.py | 1 - rida/scheduler/handlers/modules.py | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/rida/pdc.py b/rida/pdc.py index 6d85e0e8..4df62101 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -96,7 +96,6 @@ def get_variant_dict(data): return result - def variant_dict_from_str(module_str): """ :param module_str: a string to match in PDC diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index de25b72f..d383c991 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -42,14 +42,10 @@ def init(config, session, msg): build = rida.database.ModuleBuild.from_fedmsg(session, msg) pdc_session = rida.pdc.get_pdc_client_session(config) - build_data = build.json() - log.debug("Getting module from pdc with input_data=%s" % build_data) - module_info = rida.pdc.get_module(pdc_session, build_data) - + module_info = build.json() log.debug("Received module_info=%s from pdc" % module_info) - tag = rida.pdc.get_module_tag(pdc_session, module_info) - log.info("Found tag=%s for module %s-%s-%s" % (tag, build['name'], build['version'], build['release'])) + log.info("Found tag=%s for module %r" % (tag, build)) dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config) From 7be07d37b1ad5dad6453df595813dc4fc1a6fbad Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 13:07:04 -0400 Subject: [PATCH 006/106] Fix a mock that broke. --- tests/test_scheduler/test_modules/test_init.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_scheduler/test_modules/test_init.py b/tests/test_scheduler/test_modules/test_init.py index 294e1be8..bfc8878d 100644 --- a/tests/test_scheduler/test_modules/test_init.py +++ b/tests/test_scheduler/test_modules/test_init.py @@ -40,9 +40,10 @@ class TestInit(unittest.TestCase): builder = mock.Mock() KojiModuleBuilder.return_value = builder mocked_module_build = mock.Mock() - mocked_module_build.to_pdc_module_info.return_value = { + mocked_module_build.json.return_value = { 'name': 'foo', 'version': 1, + 'release': 1, } from_fedmsg.return_value = mocked_module_build From 3d60ccad20903aebcee6d0ddd7d2e4318f5339e8 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 13:17:58 -0400 Subject: [PATCH 007/106] Set up a resumed builder here. --- rida/scheduler/handlers/modules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index d383c991..219d9b60 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -67,6 +67,9 @@ def build(config, session, msg): All we do here is kick off builds of all our components. """ module_build = rida.database.ModuleBuild.from_fedmsg(session, msg) + builder = rida.builder.KojiModuleBuilder(build.name, config) + builder.buildroot_resume() + for component_build in module_build.component_builds: scmurl = "{dist_git}/rpms/{package}?#{gitref}".format( dist_git=config.dist_git_url, From 2344e03f2327f8671cea9639ae2a4b234780e3f0 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 13:24:45 -0400 Subject: [PATCH 008/106] Send builds to failed state if they don't make it to 'wait'. --- rida.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/rida.py b/rida.py index b05a8c77..8ddd79a6 100755 --- a/rida.py +++ b/rida.py @@ -104,11 +104,25 @@ def submit_build(): release=mmd.release, state="init", modulemd=yaml) db.session.add(module) db.session.commit() + rida.messaging.publish( + modname='rida', + topic='module.state.change', + msg=module.json(), # Note the state is "init" here... + backend=conf.messaging, + ) + + def failure(message, code): + # TODO, we should make some note of why it failed in a log... + module.state = rida.database.BUILD_STATES["failed"] + db.session.add(module) + db.session.commit() + return message, code + for pkgname, pkg in mmd.components.rpms.packages.items(): if pkg.get("repository") and not conf.rpms_allow_repository: - return "Custom component repositories aren't allowed", 403 + return failure("Custom component repositories aren't allowed", 403) if pkg.get("cache") and not conf.rpms_allow_cache: - return "Custom component caches aren't allowed", 403 + return failure("Custom component caches aren't allowed", 403) if not pkg.get("repository"): pkg["repository"] = conf.rpms_default_repository + pkgname if not pkg.get("cache"): @@ -117,9 +131,9 @@ def submit_build(): try: pkg["commit"] = rida.scm.SCM(pkg["repository"]).get_latest() except Exception as e: - return "Failed to get the latest commit: %s" % pkgname, 422 + return failure("Failed to get the latest commit: %s" % pkgname, 422) if not rida.scm.SCM(pkg["repository"] + "?#" + pkg["commit"]).is_available(): - return "Cannot checkout %s" % pkgname, 422 + return failure("Cannot checkout %s" % pkgname, 422) build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() @@ -131,7 +145,7 @@ def submit_build(): rida.messaging.publish( modname='rida', topic='module.state.change', - msg=module.json(), + msg=module.json(), # Note the state is "wait" here... backend=conf.messaging, ) logging.info("%s submitted build of %s-%s-%s", username, mmd.name, From 44e35c3a09b4480102fd90fb1e8fb845056e6439 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 14:01:41 -0400 Subject: [PATCH 009/106] Use module.transition to change states. --- rida.py | 4 ++-- rida/database.py | 9 +++++++++ rida/scheduler/handlers/modules.py | 7 ++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/rida.py b/rida.py index 8ddd79a6..866d217a 100755 --- a/rida.py +++ b/rida.py @@ -113,7 +113,7 @@ def submit_build(): def failure(message, code): # TODO, we should make some note of why it failed in a log... - module.state = rida.database.BUILD_STATES["failed"] + module.transition(rida.database.BUILD_STATES["failed"]) db.session.add(module) db.session.commit() return message, code @@ -137,7 +137,7 @@ def submit_build(): build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() - module.state = rida.database.BUILD_STATES["wait"] + module.transition(rida.database.BUILD_STATES["wait"]) db.session.add(module) db.session.commit() # Publish to whatever bus we're configured to connect to. diff --git a/rida/database.py b/rida/database.py index 8504bad7..ff63e3ec 100644 --- a/rida/database.py +++ b/rida/database.py @@ -38,6 +38,9 @@ from sqlalchemy.orm import ( ) from sqlalchemy.ext.declarative import declarative_base +import logging +log = logging.getLogger(__name__) + # Just like koji.BUILD_STATES, except our own codes for modules. BUILD_STATES = { @@ -143,6 +146,12 @@ class ModuleBuild(Base): raise ValueError("%r is not a module message." % msg['topic']) return session.query(cls).filter(cls.id==msg['msg']['id']).one() + def transition(self, state): + """ Record that a build has transitioned state. """ + old_state = self.state + self.state = state + log.debug("%r, state %r->%r" % (old_state, self.state)) + def json(self): return { 'id': self.id, diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 219d9b60..7c02bd73 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -55,8 +55,8 @@ def init(config, session, msg): # TODO submit build from srpm to koji # TODO: buildroot.add_artifact(build_with_dist_tags) # TODO: buildroot.ready(artifact=$artifact) - build.state = "wait" # Wait for the buildroot to be ready. - log.debug("Done with init") + build.transition(state="build") # Wait for the buildroot to be ready. + session.commit() def build(config, session, msg): @@ -80,4 +80,5 @@ def build(config, session, msg): component_build.task = builder.build(artifact_name, scmurl) component_build.state = koji.BUILD_STATES['BUILDING'] - build.state = "build" # Now wait for all of those to finish. + build.transition(state="build") # Now wait for all of those to finish. + session.commit() From fe863aa33ae3dcf31e68a8162110632ab17f44f0 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 14:05:37 -0400 Subject: [PATCH 010/106] Rename this state transition handler. --- rida/scheduler/handlers/modules.py | 8 +++++--- .../test_modules/{test_init.py => test_wait.py} | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) rename tests/test_scheduler/test_modules/{test_init.py => test_wait.py} (95%) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 7c02bd73..f6ba125e 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -32,12 +32,14 @@ import koji log = logging.getLogger(__name__) -def init(config, session, msg): - """ Called whenever a module enters the 'init' state. +def wait(config, session, msg): + """ Called whenever a module enters the 'wait' state. - We usually transition to this state when the modulebuild is first requested. + We transition to this state shortly after a modulebuild is first requested. All we do here is request preparation of the buildroot. + The kicking off of individual component builds is handled elsewhere, + in rida.schedulers.handlers.repos. """ build = rida.database.ModuleBuild.from_fedmsg(session, msg) pdc_session = rida.pdc.get_pdc_client_session(config) diff --git a/tests/test_scheduler/test_modules/test_init.py b/tests/test_scheduler/test_modules/test_wait.py similarity index 95% rename from tests/test_scheduler/test_modules/test_init.py rename to tests/test_scheduler/test_modules/test_wait.py index bfc8878d..945a0df0 100644 --- a/tests/test_scheduler/test_modules/test_init.py +++ b/tests/test_scheduler/test_modules/test_wait.py @@ -26,12 +26,12 @@ import mock import rida.scheduler.handlers.modules -class TestInit(unittest.TestCase): +class TestWait(unittest.TestCase): def setUp(self): self.config = mock.Mock() self.session = mock.Mock() - self.fn = rida.scheduler.handlers.modules.init + self.fn = rida.scheduler.handlers.modules.wait @mock.patch('rida.builder.KojiModuleBuilder') @mock.patch('rida.database.ModuleBuild.from_fedmsg') From fa91f23be513579772929858b631590d9b70ab08 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 14:27:28 -0400 Subject: [PATCH 011/106] Just formatting. --- rida.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rida.py b/rida.py index 866d217a..1c2eb01e 100755 --- a/rida.py +++ b/rida.py @@ -100,8 +100,13 @@ def submit_build(): if db.session.query(rida.database.ModuleBuild).filter_by(name=mmd.name, version=mmd.version, release=mmd.release).first(): return "Module already exists", 409 - module = rida.database.ModuleBuild(name=mmd.name, version=mmd.version, - release=mmd.release, state="init", modulemd=yaml) + module = rida.database.ModuleBuild( + name=mmd.name, + version=mmd.version, + release=mmd.release, + state="init", + modulemd=yaml, + ) db.session.add(module) db.session.commit() rida.messaging.publish( From 0ab979330581ebc10382339d74f2d47b79c15a1c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 23:20:23 -0400 Subject: [PATCH 012/106] Some more state transitions. --- rida.py | 48 ++++++++----------- rida/database.py | 53 ++++++++++++++++++++- rida/scheduler/handlers/components.py | 66 +++++++++++++++++++++++++++ rida/scheduler/handlers/modules.py | 33 ++++---------- rida/scheduler/handlers/repos.py | 63 +++++++++++++++++++++++++ rida/scheduler/main.py | 10 ++-- 6 files changed, 213 insertions(+), 60 deletions(-) create mode 100644 rida/scheduler/handlers/components.py create mode 100644 rida/scheduler/handlers/repos.py diff --git a/rida.py b/rida.py index 1c2eb01e..cb6c84e5 100755 --- a/rida.py +++ b/rida.py @@ -1,7 +1,5 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- - - # Copyright (c) 2016 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -30,6 +28,7 @@ This is the implementation of the orchestrator's public RESTful API. """ from flask import Flask, request +import flask import json import logging import modulemd @@ -38,9 +37,9 @@ import rida.auth import rida.config import rida.database import rida.logger -import rida.messaging import rida.scm import ssl +import shutil import tempfile app = Flask(__name__) @@ -50,6 +49,8 @@ app.config.from_envvar("RIDA_SETTINGS", silent=True) conf = rida.config.from_file("rida.conf") rida.logger.init_logging(conf) +log = logging.getLogger(__name__) + db = rida.database.Database(conf) @app.route("/rida/module-builds/", methods=["POST"]) @@ -77,13 +78,12 @@ def submit_build(): return "The submitted scmurl isn't allowed", 403 yaml = str() try: + td = tempfile.mkdtemp() scm = rida.scm.SCM(url, conf.scmurls) - td = tempfile.TemporaryDirectory() - cod = scm.checkout(td.name) + cod = scm.checkout(td) cofn = os.path.join(cod, (scm.name + ".yaml")) with open(cofn, "r") as mmdfile: yaml = mmdfile.read() - td.cleanup() except Exception as e: if "is not in the list of allowed SCMs" in str(e): rc = 403 @@ -92,6 +92,8 @@ def submit_build(): else: rc = 500 return str(e), rc + finally: + shutil.rmtree(td) mmd = modulemd.ModuleMetadata() try: mmd.loads(yaml) @@ -100,25 +102,19 @@ def submit_build(): if db.session.query(rida.database.ModuleBuild).filter_by(name=mmd.name, version=mmd.version, release=mmd.release).first(): return "Module already exists", 409 - module = rida.database.ModuleBuild( + module = rida.database.ModuleBuild.create( + db.session, + conf, name=mmd.name, version=mmd.version, release=mmd.release, - state="init", modulemd=yaml, ) - db.session.add(module) - db.session.commit() - rida.messaging.publish( - modname='rida', - topic='module.state.change', - msg=module.json(), # Note the state is "init" here... - backend=conf.messaging, - ) def failure(message, code): - # TODO, we should make some note of why it failed in a log... - module.transition(rida.database.BUILD_STATES["failed"]) + # TODO, we should make some note of why it failed in the db.. + log.exception(message) + module.transition(conf, rida.database.BUILD_STATES["failed"]) db.session.add(module) db.session.commit() return message, code @@ -142,26 +138,18 @@ def submit_build(): build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() - module.transition(rida.database.BUILD_STATES["wait"]) + module.transition(conf, rida.database.BUILD_STATES["wait"]) db.session.add(module) db.session.commit() - # Publish to whatever bus we're configured to connect to. - # This should notify ridad to start doing the work we just scheduled. - rida.messaging.publish( - modname='rida', - topic='module.state.change', - msg=module.json(), # Note the state is "wait" here... - backend=conf.messaging, - ) logging.info("%s submitted build of %s-%s-%s", username, mmd.name, mmd.version, mmd.release) - return json.dumps(module.json()), 201 + return flask.jsonify(module.json()), 201 @app.route("/rida/module-builds/", methods=["GET"]) def query_builds(): """Lists all tracked module builds.""" - return json.dumps([{"id": x.id, "state": x.state} + return flask.jsonify([{"id": x.id, "state": x.state} for x in db.session.query(rida.database.ModuleBuild).all()]), 200 @@ -175,7 +163,7 @@ def query_build(id): for build in db.session.query(rida.database.ComponentBuild).filter_by(module_id=id).all(): tasks[build.format + "/" + build.package] = \ str(build.task) + "/" + build.state - return json.dumps({ + return flask.jsonify({ "id": module.id, "state": module.state, "tasks": tasks diff --git a/rida/database.py b/rida/database.py index ff63e3ec..772bb50a 100644 --- a/rida/database.py +++ b/rida/database.py @@ -38,6 +38,8 @@ from sqlalchemy.orm import ( ) from sqlalchemy.ext.declarative import declarative_base +import rida.messaging + import logging log = logging.getLogger(__name__) @@ -129,6 +131,7 @@ class ModuleBuild(Base): release = Column(String, nullable=False) state = Column(Integer, nullable=False) modulemd = Column(String, nullable=False) + koji_tag = Column(String) # This gets set after 'wait' module = relationship('Module', backref='module_builds', lazy=False) @@ -146,11 +149,57 @@ class ModuleBuild(Base): raise ValueError("%r is not a module message." % msg['topic']) return session.query(cls).filter(cls.id==msg['msg']['id']).one() - def transition(self, state): + @classmethod + def create(cls, session, conf, name, version, release, modulemd): + module = cls( + name=name, + version=version, + release=release, + state="init", + modulemd=modulemd, + ) + session.add(module) + session.commit() + rida.messaging.publish( + modname='rida', + topic='module.state.change', + msg=module.json(), # Note the state is "init" here... + backend=conf.messaging, + ) + return module + + def transition(self, conf, state): """ Record that a build has transitioned state. """ old_state = self.state self.state = state - log.debug("%r, state %r->%r" % (old_state, self.state)) + log.debug("%r, state %r->%r" % (self, old_state, self.state)) + rida.messaging.publish( + modname='rida', + topic='module.state.change', + msg=self.json(), # Note the state is "init" here... + backend=conf.messaging, + ) + + @classmethod + def get_active_by_koji_tag(cls, session, koji_tag): + """ Find the ModuleBuilds in our database that should be in-flight... + ... for a given koji tag. + + There should be at most one. + """ + query = session.query(rida.database.ModuleBuild)\ + .filter_by(koji_tag=koji_tag)\ + .filter_by(state="build") + + count = query.count() + if count > 1: + raise RuntimeError("%r module builds in flight for %r" % (count, koji_tag)) + elif count == 0: + # No builds in flight scheduled by us. Just ignore this. + return None + + # Otherwise, there is exactly one module build - it must be ours. + return query.one() def json(self): return { diff --git a/rida/scheduler/handlers/components.py b/rida/scheduler/handlers/components.py new file mode 100644 index 00000000..14509361 --- /dev/null +++ b/rida/scheduler/handlers/components.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean + +""" Handlers for koji component build events on the message bus. """ + +import rida.builder +import rida.database +import rida.pdc +import logging +import koji + +log = logging.getLogger(__name__) + + +def complete(config, session, msg): + """ Called whenever a koji build completes. """ + + # First, find our ModuleBuild associated with this repo, if any. + component_build = rida.database.ComponentBuild.from_fedmsg(session, msg) + if not component_build: + template = "We have no record of {name}-{version}-{release}" + log.debug(template.format(**msg['msg'])) + return + + # Mark the state in the db. + component_build.state = koji.BUILD_STATES['COMPLETE'] + session.commit() + + # Find all of the sibling builds of this particular build. + parent = component_build.module_build + siblings = parent.component_builds + + # Are any of them still executing? + if any([c.state == koji.BUILD_STATES['BUILDING'] for c in siblings]): + # Then they're not all done yet... continue to wait + return + + # Otherwise, check to see if any failed. + if any([c.state != koji.BUILD_STATES['COMPLETE'] for c in siblings]): + # They didn't all succeed.. so mark this module build as a failure. + parent.transition(config, rida.BUILD_STATES['failed']) + session.commit() + return + + # Otherwise.. if all of the builds succeeded, then mark the module as good. + parent.transition(config, rida.BUILD_STATES['done']) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index f6ba125e..5c902c52 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -47,7 +47,13 @@ def wait(config, session, msg): module_info = build.json() log.debug("Received module_info=%s from pdc" % module_info) tag = rida.pdc.get_module_tag(pdc_session, module_info) - log.info("Found tag=%s for module %r" % (tag, build)) + log.debug("Found tag=%s for module %r" % (tag, build)) + + # Hang on to this information for later. We need to know which build is + # associated with which koji tag, so that when their repos are regenerated + # in koji we can figure out which for which module build that event is + # relevant. + build.tag = tag dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config) @@ -57,30 +63,7 @@ def wait(config, session, msg): # TODO submit build from srpm to koji # TODO: buildroot.add_artifact(build_with_dist_tags) # TODO: buildroot.ready(artifact=$artifact) - build.transition(state="build") # Wait for the buildroot to be ready. + build.transition(conf, state="build") # Wait for the buildroot to be ready. session.commit() -def build(config, session, msg): - """ Called whenever a module enters the "build" state. - - We usually transition to this state once the buildroot is ready. - - All we do here is kick off builds of all our components. - """ - module_build = rida.database.ModuleBuild.from_fedmsg(session, msg) - builder = rida.builder.KojiModuleBuilder(build.name, config) - builder.buildroot_resume() - - for component_build in module_build.component_builds: - scmurl = "{dist_git}/rpms/{package}?#{gitref}".format( - dist_git=config.dist_git_url, - package=component_build.package, - gitref=component_build.gitref, # This is the update stream - ) - artifact_name = 'TODO' - component_build.task = builder.build(artifact_name, scmurl) - component_build.state = koji.BUILD_STATES['BUILDING'] - - build.transition(state="build") # Now wait for all of those to finish. - session.commit() diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py new file mode 100644 index 00000000..74ce58c2 --- /dev/null +++ b/rida/scheduler/handlers/repos.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean + +""" Handlers for repo change events on the message bus. """ + +import rida.builder +import rida.database +import rida.pdc +import logging +import koji + +log = logging.getLogger(__name__) + + +def done(config, session, msg): + """ Called whenever koji rebuilds a repo, any repo. """ + + # First, find our ModuleBuild associated with this repo, if any. + tag = msg['msg']['tag'] + module_build = rida.database.ModuleBuild.get_active_by_koji_tag( + session, koji_tag=tag) + if not module_build: + log.debug("No module build found associated with koji tag %r" % tag) + return + + unbuilt_components = ( + component_build for component_build in module_build.component_builds + if component_build.state is None + ) + + builder = rida.builder.KojiModuleBuilder(module_build.name, config) + builder.buildroot_resume() + + for component_build in unbuilt_components: + scmurl = "{dist_git}/rpms/{package}?#{gitref}".format( + dist_git=config.dist_git_url, + package=component_build.package, + gitref=component_build.gitref, # This is the update stream + ) + artifact_name = 'TODO' + component_build.state = koji.BUILD_STATES['BUILDING'] + component_build.task = builder.build(artifact_name, scmurl) + session.commit() diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 5ff770fa..ea0cf1eb 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -76,8 +76,10 @@ class Messaging(threading.Thread): koji.BUILD_STATES["BUILDING"]: lambda x: x } on_module_change = { - rida.BUILD_STATES["init"]: rida.scheduler.handlers.modules.init, + rida.BUILD_STATES["wait"]: rida.scheduler.handlers.modules.wait, } + # Only one kind of repo change event... + on_repo_change = rida.scheduler.handlers.repos.done, def sanity_check(self): """ On startup, make sure our implementation is sane. """ @@ -107,8 +109,10 @@ class Messaging(threading.Thread): log.debug(msg) # Choose a handler for this message - if '.buildsys.build.state.change' in msg['topic']: - handler = self.on_build_change[msg['msg']['init']] + if '.buildsys.repo.done' in msg['topic']: + handler = self.on_repo_change + elif '.buildsys.build.state.change' in msg['topic']: + handler = self.on_build_change[msg['msg']['new']] elif '.rida.module.state.change' in msg['topic']: handler = self.on_module_change[module_build_state_from_msg(msg)] else: From e1c9cde9a566d057824c2024caf36c3a9b2fc639 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 23:51:28 -0400 Subject: [PATCH 013/106] Fix to config.polling_interval. --- rida/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/config.py b/rida/config.py index d643a744..3386d8cd 100644 --- a/rida/config.py +++ b/rida/config.py @@ -64,6 +64,7 @@ def from_file(filename=None): conf.db = default.get("db") conf.system = default.get("system") conf.messaging = default.get("messaging") + conf.polling_interval = int(default.get("polling_interval")) conf.pdc_url = default.get("pdc_url") conf.pdc_insecure = default.get("pdc_insecure") conf.pdc_develop = default.get("pdc_develop") @@ -329,4 +330,3 @@ class Config(object): def log_level(self, s): level = str(s).lower() self._log_level = logger.str_to_log_level(level) - From 9925c238aac7aa12654326ddfae2027a5cd1e996 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 23:51:47 -0400 Subject: [PATCH 014/106] Another classmethod to query with, and a bugfix! --- rida/database.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index 772bb50a..505d3794 100644 --- a/rida/database.py +++ b/rida/database.py @@ -180,6 +180,11 @@ class ModuleBuild(Base): backend=conf.messaging, ) + @classmethod + def by_state(cls, session, state): + return session.query(rida.database.ModuleBuild)\ + .filter_by(state=BUILD_STATES[state]).all() + @classmethod def get_active_by_koji_tag(cls, session, koji_tag): """ Find the ModuleBuilds in our database that should be in-flight... @@ -189,7 +194,7 @@ class ModuleBuild(Base): """ query = session.query(rida.database.ModuleBuild)\ .filter_by(koji_tag=koji_tag)\ - .filter_by(state="build") + .filter_by(state=BUILD_STATES["build"]) count = query.count() if count > 1: From 2a866eaf91c12babe6ededd8d957e63f150db2f7 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 23:53:36 -0400 Subject: [PATCH 015/106] Some start to the polling thread. --- rida/scheduler/main.py | 70 ++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index ea0cf1eb..f46459be 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -34,12 +34,14 @@ import inspect import logging import os import threading +import time import rida.config -import rida.logging +import rida.logger import rida.messaging +import rida.scheduler.handlers.components import rida.scheduler.handlers.modules -#import rida.scheduler.handlers.builds +import rida.scheduler.handlers.repos import sys import koji @@ -62,7 +64,6 @@ else: # TODO: Set the build state to failed if the module build fails. def module_build_state_from_msg(msg): - state = int(msg['msg']['state']) # TODO better handling assert state in rida.BUILD_STATES.values(), "state=%s(%s) is not in %s" % (state, type(state), rida.BUILD_STATES.values()) @@ -70,16 +71,19 @@ def module_build_state_from_msg(msg): class Messaging(threading.Thread): - # These are our main lookup tables for figuring out what to run in response - # to what messaging events. - on_build_change = { - koji.BUILD_STATES["BUILDING"]: lambda x: x - } - on_module_change = { - rida.BUILD_STATES["wait"]: rida.scheduler.handlers.modules.wait, - } - # Only one kind of repo change event... - on_repo_change = rida.scheduler.handlers.repos.done, + def __init__(self, *args, **kwargs): + super(Messaging, self).__init__(*args, **kwargs) + + # These are our main lookup tables for figuring out what to run in response + # to what messaging events. + self.on_build_change = { + koji.BUILD_STATES["BUILDING"]: lambda x: x, + } + self.on_module_change = { + rida.BUILD_STATES["wait"]: rida.scheduler.handlers.modules.wait, + } + # Only one kind of repo change event, though... + self.on_repo_change = rida.scheduler.handlers.repos.done def sanity_check(self): """ On startup, make sure our implementation is sane. """ @@ -105,8 +109,7 @@ class Messaging(threading.Thread): # TODO: Act on these things somehow # TODO: Emit messages about doing so for msg in rida.messaging.listen(backend=config.messaging): - log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic'])) - log.debug(msg) + log.debug("received %r, %r" % (msg['msg_id'], msg['topic'])) # Choose a handler for this message if '.buildsys.repo.done' in msg['topic']: @@ -121,22 +124,43 @@ class Messaging(threading.Thread): # Execute our chosen handler with rida.database.Database(config) as session: + log.info("Executing handler %r" % handler) handler(config, session, msg) class Polling(threading.Thread): def run(self): while True: - # TODO: Check for module builds in the wait state - # TODO: Check component builds in the open state - # TODO: Check for modules that can be set to done/failed - # TODO: Act on these things somehow - # TODO: Emit messages about doing so - # TODO: Sleep for a configuration-determined interval - pass + log.info("Polling thread sleeping, %rs" % config.polling_interval) + time.sleep(config.polling_interval) + with rida.database.Database(config) as session: + self.process_waiting_module_builds(session) + with rida.database.Database(config) as session: + self.process_open_component_builds(session) + with rida.database.Database(config) as session: + self.process_lingering_module_builds(session) + + def process_waiting_module_builds(self, session): + log.info("Looking for module builds stuck in the wait state.") + builds = rida.database.ModuleBuild.by_state(session, "wait") + # TODO -- do throttling calculation here... + log.info(" %r module builds in the wait state..." % len(builds)) + for build in builds: + # Fake a message to kickstart the build anew + msg = { + 'topic': '.module.build.state.change', + 'msg': build.json(), + } + rida.scheduler.handlers.modules.wait(config, session, msg) + + def process_open_component_builds(self, session): + log.warning("process_open_component_builds is not yet implemented...") + + def process_lingering_module_builds(self, session): + log.warning("process_lingering_module_builds is not yet implemented...") def main(): - rida.logging.init_logging(config) + rida.logger.init_logging(config) log.info("Starting ridad.") try: messaging_thread = Messaging() From fd5740341137eb20e9e483eed057603da677f36b Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 23:54:43 -0400 Subject: [PATCH 016/106] Typofix. --- rida/scheduler/handlers/modules.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 5c902c52..fa77bccf 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -63,7 +63,5 @@ def wait(config, session, msg): # TODO submit build from srpm to koji # TODO: buildroot.add_artifact(build_with_dist_tags) # TODO: buildroot.ready(artifact=$artifact) - build.transition(conf, state="build") # Wait for the buildroot to be ready. + build.transition(config, state="build") # Wait for the buildroot to be ready. session.commit() - - From af86b0a4fc309dcde06c2d4e2c5496265576c125 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 00:15:58 -0400 Subject: [PATCH 017/106] Some test coverage for the repo done code. --- .../test_wait.py => test_module_wait.py} | 4 +- tests/test_scheduler/test_modules/__init__.py | 0 tests/test_scheduler/test_repo_done.py | 70 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) rename tests/test_scheduler/{test_modules/test_wait.py => test_module_wait.py} (95%) delete mode 100644 tests/test_scheduler/test_modules/__init__.py create mode 100644 tests/test_scheduler/test_repo_done.py diff --git a/tests/test_scheduler/test_modules/test_wait.py b/tests/test_scheduler/test_module_wait.py similarity index 95% rename from tests/test_scheduler/test_modules/test_wait.py rename to tests/test_scheduler/test_module_wait.py index 945a0df0..5f60c3f6 100644 --- a/tests/test_scheduler/test_modules/test_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -26,7 +26,7 @@ import mock import rida.scheduler.handlers.modules -class TestWait(unittest.TestCase): +class TestModuleWait(unittest.TestCase): def setUp(self): self.config = mock.Mock() @@ -36,7 +36,7 @@ class TestWait(unittest.TestCase): @mock.patch('rida.builder.KojiModuleBuilder') @mock.patch('rida.database.ModuleBuild.from_fedmsg') @mock.patch('rida.pdc.get_pdc_client_session') - def test_init_basic(self, pdc, from_fedmsg, KojiModuleBuilder): + def test_wait_basic(self, pdc, from_fedmsg, KojiModuleBuilder): builder = mock.Mock() KojiModuleBuilder.return_value = builder mocked_module_build = mock.Mock() diff --git a/tests/test_scheduler/test_modules/__init__.py b/tests/test_scheduler/test_modules/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py new file mode 100644 index 00000000..d010ab0a --- /dev/null +++ b/tests/test_scheduler/test_repo_done.py @@ -0,0 +1,70 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean + +import unittest +import mock + +import rida.scheduler.handlers.repos + + +class TestRepoDone(unittest.TestCase): + + def setUp(self): + self.config = mock.Mock() + self.config.dist_git_url = 'dist_git_url' + + self.session = mock.Mock() + self.fn = rida.scheduler.handlers.repos.done + + @mock.patch('rida.database.ModuleBuild.get_active_by_koji_tag') + def test_no_match(self, get_active_by_koji_tag): + """ Test that when a repo msg hits us and we have no match, + that we do nothing gracefully. + """ + get_active_by_koji_tag.return_value = None + msg = { + 'topic': 'org.fedoraproject.prod.buildsys.repo.done', + 'msg': {'tag': 'no matches for this...'}, + } + self.fn(config=self.config, session=self.session, msg=msg) + + @mock.patch('rida.builder.KojiModuleBuilder.build') + @mock.patch('rida.builder.KojiModuleBuilder.buildroot_resume') + @mock.patch('rida.database.ModuleBuild.get_active_by_koji_tag') + def test_a_single_match(self, get_active_by_koji_tag, resume, build_fn): + """ Test that when a repo msg hits us and we have no match, + that we do nothing gracefully. + """ + component_build = mock.Mock() + component_build.package = 'foo' + component_build.gitref = 'beef' + component_build.state = None + module_build = mock.Mock() + module_build.component_builds = [component_build] + + get_active_by_koji_tag.return_value = module_build + msg = { + 'topic': 'org.fedoraproject.prod.buildsys.repo.done', + 'msg': {'tag': 'no matches for this...'}, + } + self.fn(config=self.config, session=self.session, msg=msg) + build_fn.assert_called_once_with('TODO', 'dist_git_url/rpms/foo?#beef') From 2871ca95371ee35d1d479b2fc560c7710f32b356 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 00:20:08 -0400 Subject: [PATCH 018/106] Remove some TODOs which are done. --- rida/scheduler/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index f46459be..8815f443 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -58,10 +58,6 @@ else: # production config = rida.config.from_file() -# TODO: Utilized rida.builder to prepare the buildroots and build components. -# TODO: Set the build state to build once the module build is started. -# TODO: Set the build state to done once the module build is done. -# TODO: Set the build state to failed if the module build fails. def module_build_state_from_msg(msg): state = int(msg['msg']['state']) From 0b369baf665a5cc6c7bdf73074f9b87030285115 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 00:53:59 -0400 Subject: [PATCH 019/106] Rename to build_id, which is more accurate. --- rida/builder.py | 8 ++++---- rida/database.py | 10 ++++++++-- rida/scheduler/handlers/repos.py | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 3d267df5..b9155955 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -231,7 +231,7 @@ class KojiModuleBuilder(GenericBuilder): def build(self, artifact_name, source): """ :param source : scmurl to spec repository - :return koji taskid + :return koji build id """ if not self.__prep: raise RuntimeError("Buildroot is not prep-ed") @@ -239,9 +239,9 @@ class KojiModuleBuilder(GenericBuilder): if '://' not in source: raise NotImplementedError("Only scm url is currently supported, got source='%s'" % source) self._koji_whitelist_packages([artifact_name,]) - task_id = self.koji_session.build(source, self.module_target['name']) - print("Building %s (taskid=%s)." % (source, task_id)) - return task_id + build_id = self.koji_session.build(source, self.module_target['name']) + print("Building %s (build_id=%s)." % (source, build_id)) + return build_id def _get_tag(self, tag, strict=True): if isinstance(tag, dict): diff --git a/rida/database.py b/rida/database.py index 505d3794..c4d098ab 100644 --- a/rida/database.py +++ b/rida/database.py @@ -232,19 +232,25 @@ class ComponentBuild(Base): package = Column(String, nullable=False) # XXX: Consider making this a proper ENUM format = Column(String, nullable=False) - task = Column(Integer) + build_id = Column(Integer) # This is the id of the build in koji # XXX: Consider making this a proper ENUM (or an int) state = Column(String) module_id = Column(Integer, ForeignKey('module_builds.id'), nullable=False) module_build = relationship('ModuleBuild', backref='component_builds', lazy=False) + @classmethod + def from_fedmsg(cls, session, msg): + if '.buildsys.build.state.change' not in msg['topic']: + raise ValueError("%r is not a koji message." % msg['topic']) + return session.query(cls).filter(cls.build_id==msg['msg']['id']).one() + def json(self): return { 'id': self.id, 'package': self.package, 'format': self.format, - 'task': self.task, + 'build_id': self.build_id, 'state': self.state, 'module_build': self.module_id, } diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 74ce58c2..c14570fd 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -59,5 +59,5 @@ def done(config, session, msg): ) artifact_name = 'TODO' component_build.state = koji.BUILD_STATES['BUILDING'] - component_build.task = builder.build(artifact_name, scmurl) + component_build.build_id = builder.build(artifact_name, scmurl) session.commit() From c4ae0c3a925721c725bc79b8e9b05fed9071b64a Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 00:54:10 -0400 Subject: [PATCH 020/106] Console will do. --- rida.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida.conf b/rida.conf index e7d4c84e..e500e35a 100644 --- a/rida.conf +++ b/rida.conf @@ -30,7 +30,7 @@ ssl_ca_certificate_file = cacert.pem pkgdb_api_url = https://admin.stg.fedoraproject.org/pkgdb/api # Available backends are: console, file, journal. -log_backend = journal +log_backend = console # Path to log file when log_backend is set to "file". log_file = rida.log From 88b7ca50113cd0b7142004ecd3962737436fe3e1 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 00:54:47 -0400 Subject: [PATCH 021/106] Get the sanity_check running. --- rida/scheduler/handlers/components.py | 21 +++++++++++++++----- rida/scheduler/main.py | 28 +++++++++++++++++---------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/rida/scheduler/handlers/components.py b/rida/scheduler/handlers/components.py index 14509361..fcd1c5f6 100644 --- a/rida/scheduler/handlers/components.py +++ b/rida/scheduler/handlers/components.py @@ -23,17 +23,18 @@ """ Handlers for koji component build events on the message bus. """ +import logging + import rida.builder import rida.database import rida.pdc -import logging + import koji log = logging.getLogger(__name__) - -def complete(config, session, msg): - """ Called whenever a koji build completes. """ +def _finalize(config, session, msg, state): + """ Called whenever a koji build completes or fails. """ # First, find our ModuleBuild associated with this repo, if any. component_build = rida.database.ComponentBuild.from_fedmsg(session, msg) @@ -43,7 +44,7 @@ def complete(config, session, msg): return # Mark the state in the db. - component_build.state = koji.BUILD_STATES['COMPLETE'] + component_build.state = state session.commit() # Find all of the sibling builds of this particular build. @@ -64,3 +65,13 @@ def complete(config, session, msg): # Otherwise.. if all of the builds succeeded, then mark the module as good. parent.transition(config, rida.BUILD_STATES['done']) + + +def complete(config, session, msg): + return _finalize(config, session, msg, state=koji.BUILD_STATES['COMPLETE']) + +def failed(config, session, msg): + return _finalize(config, session, msg, state=koji.BUILD_STATES['FAILED']) + +def canceled(config, session, msg): + return _finalize(config, session, msg, state=koji.BUILD_STATES['CANCELED']) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 8815f443..02e1d9b8 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -72,11 +72,21 @@ class Messaging(threading.Thread): # These are our main lookup tables for figuring out what to run in response # to what messaging events. + NO_OP = lambda config, session, msg: True self.on_build_change = { - koji.BUILD_STATES["BUILDING"]: lambda x: x, + koji.BUILD_STATES["BUILDING"]: NO_OP, + koji.BUILD_STATES["COMPLETE"]: rida.scheduler.handlers.components.complete, + koji.BUILD_STATES["FAILED"]: rida.scheduler.handlers.components.failed, + koji.BUILD_STATES["CANCELED"]: rida.scheduler.handlers.components.canceled, + koji.BUILD_STATES["DELETED"]: NO_OP, } self.on_module_change = { + rida.BUILD_STATES["init"]: NO_OP, rida.BUILD_STATES["wait"]: rida.scheduler.handlers.modules.wait, + rida.BUILD_STATES["build"]: NO_OP, + rida.BUILD_STATES["failed"]: NO_OP, + rida.BUILD_STATES["done"]: NO_OP, + rida.BUILD_STATES["ready"]: NO_OP, } # Only one kind of repo change event, though... self.on_repo_change = rida.scheduler.handlers.repos.done @@ -85,25 +95,23 @@ class Messaging(threading.Thread): """ On startup, make sure our implementation is sane. """ # Ensure we have every state covered for state in rida.BUILD_STATES: - if state not in self.on_module_change: + if rida.BUILD_STATES[state] not in self.on_module_change: raise KeyError("Module build states %r not handled." % state) for state in koji.BUILD_STATES: - if state not in self.on_build_change: + if koji.BUILD_STATES[state] not in self.on_build_change: raise KeyError("Koji build states %r not handled." % state) all_fns = self.on_build_change.items() + self.on_module_change.items() for key, callback in all_fns: - expected = ['conf', 'db', 'msg'] - argspec = inspect.getargspec(callback) + expected = ['config', 'session', 'msg'] + argspec = inspect.getargspec(callback)[0] if argspec != expected: raise ValueError("Callback %r, state %r has argspec %r!=%r" % ( callback, key, argspec, expected)) def run(self): - #self.sanity_check() - # TODO: Check for modules that can be set to done/failed - # TODO: Act on these things somehow - # TODO: Emit messages about doing so + self.sanity_check() + for msg in rida.messaging.listen(backend=config.messaging): log.debug("received %r, %r" % (msg['msg_id'], msg['topic'])) @@ -120,7 +128,7 @@ class Messaging(threading.Thread): # Execute our chosen handler with rida.database.Database(config) as session: - log.info("Executing handler %r" % handler) + log.info(" %r: %s, %s" % (handler, msg['topic'], msg['msg_id'])) handler(config, session, msg) class Polling(threading.Thread): From dee31ed512fc8acc06c817ff21f71c8ab5337f9f Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 01:02:07 -0400 Subject: [PATCH 022/106] Typofix. --- rida/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index c4d098ab..daa6e852 100644 --- a/rida/database.py +++ b/rida/database.py @@ -147,7 +147,7 @@ class ModuleBuild(Base): def from_fedmsg(cls, session, msg): if '.module.' not in msg['topic']: raise ValueError("%r is not a module message." % msg['topic']) - return session.query(cls).filter(cls.id==msg['msg']['id']).one() + return session.query(cls).filter(cls.id==msg['msg']['build_id']).one() @classmethod def create(cls, session, conf, name, version, release, modulemd): From 285cf212c1aeec65c5d4e70b84ebc054f9093986 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 01:02:16 -0400 Subject: [PATCH 023/106] Top-level error handling. --- rida/scheduler/main.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 02e1d9b8..1e2dc87a 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -113,23 +113,29 @@ class Messaging(threading.Thread): self.sanity_check() for msg in rida.messaging.listen(backend=config.messaging): - log.debug("received %r, %r" % (msg['msg_id'], msg['topic'])) + try: + self.process_message(msg) + except Exception: + log.exception("Failed while handling %r" % msg['msg_id']) - # Choose a handler for this message - if '.buildsys.repo.done' in msg['topic']: - handler = self.on_repo_change - elif '.buildsys.build.state.change' in msg['topic']: - handler = self.on_build_change[msg['msg']['new']] - elif '.rida.module.state.change' in msg['topic']: - handler = self.on_module_change[module_build_state_from_msg(msg)] - else: - log.debug("Unhandled message...") - continue + def process_message(self, msg): + log.debug("received %r, %r" % (msg['msg_id'], msg['topic'])) - # Execute our chosen handler - with rida.database.Database(config) as session: - log.info(" %r: %s, %s" % (handler, msg['topic'], msg['msg_id'])) - handler(config, session, msg) + # Choose a handler for this message + if '.buildsys.repo.done' in msg['topic']: + handler = self.on_repo_change + elif '.buildsys.build.state.change' in msg['topic']: + handler = self.on_build_change[msg['msg']['new']] + elif '.rida.module.state.change' in msg['topic']: + handler = self.on_module_change[module_build_state_from_msg(msg)] + else: + log.debug("Unhandled message...") + return + + # Execute our chosen handler + with rida.database.Database(config) as session: + log.info(" %r: %s, %s" % (handler, msg['topic'], msg['msg_id'])) + handler(config, session, msg) class Polling(threading.Thread): def run(self): From 5d914e8c355559979b11dbed26432a37844ff1ec Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 19:27:50 -0400 Subject: [PATCH 024/106] I got these two backwards. --- rida/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/database.py b/rida/database.py index daa6e852..501c0c2f 100644 --- a/rida/database.py +++ b/rida/database.py @@ -147,7 +147,7 @@ class ModuleBuild(Base): def from_fedmsg(cls, session, msg): if '.module.' not in msg['topic']: raise ValueError("%r is not a module message." % msg['topic']) - return session.query(cls).filter(cls.id==msg['msg']['build_id']).one() + return session.query(cls).filter(cls.id==msg['msg']['id']).one() @classmethod def create(cls, session, conf, name, version, release, modulemd): @@ -243,7 +243,7 @@ class ComponentBuild(Base): def from_fedmsg(cls, session, msg): if '.buildsys.build.state.change' not in msg['topic']: raise ValueError("%r is not a koji message." % msg['topic']) - return session.query(cls).filter(cls.build_id==msg['msg']['id']).one() + return session.query(cls).filter(cls.build_id==msg['msg']['build_id']).one() def json(self): return { From d020cd74a6713cb9f567ed73a84704ecafdf1aed Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 19:28:14 -0400 Subject: [PATCH 025/106] Some nice logging to see the message on which we failed. --- rida/scheduler/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 1e2dc87a..f3d47bec 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -33,6 +33,7 @@ proper scheduling component builds in the supported build systems. import inspect import logging import os +import pprint import threading import time @@ -117,6 +118,7 @@ class Messaging(threading.Thread): self.process_message(msg) except Exception: log.exception("Failed while handling %r" % msg['msg_id']) + log.info(pprint.pformat(msg)) def process_message(self, msg): log.debug("received %r, %r" % (msg['msg_id'], msg['topic'])) From cce333bd73f1d983a489f50b771456342d5cab41 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 19:28:26 -0400 Subject: [PATCH 026/106] Move sleep to the end to tighten my debugging loop. --- rida/scheduler/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index f3d47bec..e64e4b47 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -142,14 +142,14 @@ class Messaging(threading.Thread): class Polling(threading.Thread): def run(self): while True: - log.info("Polling thread sleeping, %rs" % config.polling_interval) - time.sleep(config.polling_interval) with rida.database.Database(config) as session: self.process_waiting_module_builds(session) with rida.database.Database(config) as session: self.process_open_component_builds(session) with rida.database.Database(config) as session: self.process_lingering_module_builds(session) + log.info("Polling thread sleeping, %rs" % config.polling_interval) + time.sleep(config.polling_interval) def process_waiting_module_builds(self, session): log.info("Looking for module builds stuck in the wait state.") From 0db0d1d50a419fb04cbd3c665b9e6f49e4747917 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 19:39:42 -0400 Subject: [PATCH 027/106] Add strict behavior to the pdc interface to make debugging easier. --- rida/pdc.py | 40 ++++++++++++++++-------------- rida/scheduler/handlers/modules.py | 8 +++--- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/rida/pdc.py b/rida/pdc.py index 4df62101..58c6dff3 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -113,7 +113,7 @@ def variant_dict_from_str(module_str): return module_info -def get_module(session, module_info): +def get_module(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info: pdc variant_dict, str, mmd or module dict @@ -122,23 +122,27 @@ def get_module(session, module_info): module_info = get_variant_dict(module_info) - module_info = session['unreleasedvariants'](page_size=-1, **module_info) - assert len(module_info) <= 1 + retval = session['unreleasedvariants'](page_size=-1, **module_info) + assert len(retval) <= 1 - if not module_info: - return None + # Error handling + if not retval: + if strict: + raise ValueError("Failed to find module in PDC %r" % module_info) + else: + return None - return module_info[0] + return retval[0] -def get_module_tag(session, module_info): +def get_module_tag(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info: list of module_info dicts :return: koji tag string """ - return get_module(session, module_info)['koji_tag'] + return get_module(session, module_info, strict=strict)['koji_tag'] -def module_depsolving_wrapper(session, module_list): +def module_depsolving_wrapper(session, module_list, strict=False): """ :param session : PDCClient instance :param module_list: list of module_info dicts @@ -147,11 +151,11 @@ def module_depsolving_wrapper(session, module_list): # TODO: implement this # Make sure that these are dicts from PDC ... ensures all values - module_infos = [get_module(session, module) for module in module_list] + module_infos = [get_module(session, module, strict=strict) for module in module_list] return module_infos -def get_module_dependencies(session, module_info): +def get_module_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance :param module_infos : a dict containing filters for pdc @@ -161,14 +165,14 @@ def get_module_dependencies(session, module_info): # XXX get definitive list of modules deps = [] - module_info = get_module(session, module_info) - if module_info.get('runtime_deps'): + module_info = get_module(session, module_info, strict=strict) + if module_info and module_info.get('runtime_deps'): deps = [x['dependency'] for x in module_info['runtime_deps']] - deps = module_depsolving_wrapper(session, deps) + deps = module_depsolving_wrapper(session, deps, strict=strict) return deps -def get_module_build_dependencies(session, module_info): +def get_module_build_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info : a dict containing filters for pdc @@ -179,9 +183,9 @@ def get_module_build_dependencies(session, module_info): # XXX get definitive list of modules deps = [] - module_info = get_module(session, module_info) - if module_info.get('build_deps'): + module_info = get_module(session, module_info, strict=strict) + if module_info and module_info.get('build_deps'): deps = [x['dependency'] for x in module_info['build_deps']] - deps = module_depsolving_wrapper(session, deps) + deps = module_depsolving_wrapper(session, deps, strict=strict) return deps diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index fa77bccf..7089f535 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -42,11 +42,11 @@ def wait(config, session, msg): in rida.schedulers.handlers.repos. """ build = rida.database.ModuleBuild.from_fedmsg(session, msg) - pdc_session = rida.pdc.get_pdc_client_session(config) - module_info = build.json() - log.debug("Received module_info=%s from pdc" % module_info) - tag = rida.pdc.get_module_tag(pdc_session, module_info) + log.info("Found module_info=%s from message" % module_info) + + pdc_session = rida.pdc.get_pdc_client_session(config) + tag = rida.pdc.get_module_tag(pdc_session, module_info, strict=True) log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is From a04d10b0f3f233e3efbaa6ac7705efe5e43763fb Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 19:39:50 -0400 Subject: [PATCH 028/106] Remove unused import. --- rida/scheduler/handlers/modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 7089f535..51eccf92 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -27,7 +27,6 @@ import rida.builder import rida.database import rida.pdc import logging -import koji log = logging.getLogger(__name__) From f23094318199c4869851092dc54b4072e67948d1 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 19:59:05 -0400 Subject: [PATCH 029/106] Use the koji tag name from PDC instead of a derived one. --- rida/builder.py | 39 ++++++++++++------------------ rida/scheduler/handlers/modules.py | 2 +- rida/scheduler/handlers/repos.py | 2 +- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index b9155955..eb5e21f4 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -128,7 +128,7 @@ class GenericBuilder: class Builder: """Wrapper class""" - def __new__(cls, module, backend, config): + def __new__(cls, module, backend, config, **extra): """ :param module : a module string e.g. 'testmodule-1.0' :param backend: a string representing backend e.g. 'koji' @@ -136,7 +136,7 @@ class Builder: """ if backend == "koji": - return KojiModuleBuilder(module=module, config=config) + return KojiModuleBuilder(module=module, config=config, **extra) else: raise ValueError("Builder backend='%s' not recognized" % backend) @@ -145,7 +145,7 @@ class KojiModuleBuilder(GenericBuilder): backend = "koji" - def __init__(self, module, config): + def __init__(self, module, config, tag_name): """ :param koji_profile: koji profile to be used """ @@ -174,23 +174,23 @@ class KojiModuleBuilder(GenericBuilder): self.arches = config.koji_arches - self.module_tag = self._get_module_tag_name() - self.module_build_tag = self._get_module_build_tag_name() - self.module_target = self._get_module_target_name() + self.module_tag = tag_name + self.module_build_tag = "%s-build" % tag_name + self.module_target = tag_name def buildroot_resume(self): # XXX: experimental """ Resume existing buildroot. Sets __prep=True """ - chktag = self.koji_session.getTag(self._get_module_tag_name()) + chktag = self.koji_session.getTag(self.module_tag) if not chktag: - raise SystemError("Tag %s doesn't exist" % self._get_module_tag_name()) - chkbuildtag = self.koji_session.getTag(self._get_module_build_tag_name()) + raise SystemError("Tag %s doesn't exist" % self.module_tag) + chkbuildtag = self.koji_session.getTag(self.module_build_tag) if not chkbuildtag: - raise SystemError("Build Tag %s doesn't exist" % self._get_module_build_tag_name()) - chktarget = self.koji_session.getBuildTarget(self._get_module_target_name()) + raise SystemError("Build Tag %s doesn't exist" % self.module_build_tag) + chktarget = self.koji_session.getBuildTarget(self.module_build_tag) if not chktarget: - raise SystemError("Target %s doesn't exist" % self._get_module_target_name()) + raise SystemError("Target %s doesn't exist" % self.module_target) self.module_tag = chktag self.module_build_tag = chkbuildtag self.module_target = chktarget @@ -201,14 +201,14 @@ class KojiModuleBuilder(GenericBuilder): :param module_deps_tags: a tag names of our build requires :param module_deps_tags: a tag names of our build requires """ - self.module_tag = self._koji_create_tag(self._get_module_tag_name(), perm="admin") # returns tag obj - self.module_build_tag = self._koji_create_tag(self._get_module_build_tag_name(), self.arches, perm="admin") + self.module_tag = self._koji_create_tag(self.module_tag, perm="admin") # returns tag obj + self.module_build_tag = self._koji_create_tag(self.module_build_tag, self.arches, perm="admin") groups = KOJI_DEFAULT_GROUPS # TODO: read from config if groups: self._koji_add_groups_to_tag(self.module_build_tag, groups) - self.module_target = self._koji_add_target(self._get_module_target_name(), self.module_build_tag, self.module_tag) + self.module_target = self._koji_add_target(self.module_target, self.module_build_tag, self.module_tag) self.__prep = True def buildroot_add_dependency(self, dependencies): @@ -312,15 +312,6 @@ class KojiModuleBuilder(GenericBuilder): self._lock_tag(tag_name, perm) return self._get_tag(tag_name) - def _get_module_target_name(self): - return self.module_str - - def _get_module_tag_name(self): - return self.module_str - - def _get_module_build_tag_name(self): - return "%s-build" % self._get_module_tag_name() - def _get_component_owner(self, package): user = self.koji_session.getLoggedInUser()['name'] return user diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 51eccf92..e144e81e 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -55,7 +55,7 @@ def wait(config, session, msg): build.tag = tag dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) - builder = rida.builder.KojiModuleBuilder(build.name, config) + builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) builder.buildroot_add_dependency(dependencies) build.buildroot_task_id = builder.buildroot_prep() # TODO: build srpm with dist_tag macros diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index c14570fd..e6af1d0e 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -48,7 +48,7 @@ def done(config, session, msg): if component_build.state is None ) - builder = rida.builder.KojiModuleBuilder(module_build.name, config) + builder = rida.builder.KojiModuleBuilder(module_build.name, config, tag_name=tag) builder.buildroot_resume() for component_build in unbuilt_components: From cf4eadbf9926962ec38966fc5b9b63df44c08be9 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 20:17:57 -0400 Subject: [PATCH 030/106] Some work to get koji profiles working. --- rida.conf | 6 ++++-- rida/builder.py | 43 +++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/rida.conf b/rida.conf index e500e35a..aaeda2c3 100644 --- a/rida.conf +++ b/rida.conf @@ -1,8 +1,10 @@ [DEFAULT] system = koji messaging = fedmsg -koji_config = /etc/rida/koji.conf -koji_profile = koji +koji_config = ~/.koji/config +# See https://fedoraproject.org/wiki/Koji/WritingKojiCode#Profiles +koji_profile = staging +koji_arches = ["x86_64"] db = sqlite:///rida.db pdc_url = http://fed-mod.org:8000/rest_api/v1 pdc_insecure = True diff --git a/rida/builder.py b/rida/builder.py index eb5e21f4..0b2138e6 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- - - # Copyright (c) 2016 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -30,9 +28,14 @@ # their tag names. # TODO: Ensure the RPM %dist tag is set according to the policy. -import koji from abc import ABCMeta, abstractmethod +import logging +import os + from kobo.shortcuts import run +import koji + +log = logging.getLogger(__name__) # TODO: read defaults from rida's config KOJI_DEFAULT_GROUPS = { @@ -152,25 +155,41 @@ class KojiModuleBuilder(GenericBuilder): self.module_str = module self.__prep = False self._koji_profile_name = config.koji_profile + log.debug("Using koji profile %r" % self._koji_profile_name) self.koji_module = koji.get_profile_module(self._koji_profile_name) opts = {} - krbservice = getattr(self.koji_module.config, "krbservice", None) + koji_config = self.koji_module.config + + krbservice = getattr(koji_config, "krbservice", None) if krbservice: opts["krbservice"] = krbservice - self.koji_session = koji.ClientSession(self.koji_module.config.server, opts=opts) + address = koji_config.server + log.info("Connecting to koji %r, %r" % (address, opts)) + self.koji_session = koji.ClientSession(address, opts=opts) - if self.koji_module.config.authtype == "kerberos": - keytab = getattr(self.koji_module.config, "keytab", None) - principal = getattr(self.koji_module.config, "principal", None) + authtype = koji_config.authtype + if authtype == "kerberos": + keytab = getattr(koji_config, "keytab", None) + principal = getattr(koji_config, "principal", None) if keytab and principal: - self.koji_session.krb_login(principal=principal, keytab=keytab, proxyuser=None) + self.koji_session.krb_login( + principal=principal, + keytab=keytab, + proxyuser=None, + ) else: self.koji_session.krb_login() - - elif self.koji_module.config.authtype == "ssl": - self.koji_session.ssl_login(self.koji_module.config.cert, None, self.koji_module.serverca, proxyuser=None) + elif authtype == "ssl": + self.koji_session.ssl_login( + os.path.expanduser(koji_config.cert), + None, + os.path.expanduser(koji_config.serverca), + proxyuser=None, + ) + else: + raise ValueError("Unrecognized koji authtype %r" % authtype) self.arches = config.koji_arches From d6fd3e05a905aabf0b9a8ed629deab5046a90bf6 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 20:18:12 -0400 Subject: [PATCH 031/106] Need to prep before adding deps (tags must exist before modifying them..) --- rida/scheduler/handlers/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index e144e81e..d5479f39 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -56,8 +56,8 @@ def wait(config, session, msg): dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) - builder.buildroot_add_dependency(dependencies) build.buildroot_task_id = builder.buildroot_prep() + builder.buildroot_add_dependency(dependencies) # TODO: build srpm with dist_tag macros # TODO submit build from srpm to koji # TODO: buildroot.add_artifact(build_with_dist_tags) From c4e01cea405716e1e30fed243e9d8d7458c9facb Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 20:23:16 -0400 Subject: [PATCH 032/106] Error out early if this is absent. --- rida/builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rida/builder.py b/rida/builder.py index 0b2138e6..5cac5af2 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -192,6 +192,8 @@ class KojiModuleBuilder(GenericBuilder): raise ValueError("Unrecognized koji authtype %r" % authtype) self.arches = config.koji_arches + if not self.arches: + raise ValueError("No koji_arches specified in the config.") self.module_tag = tag_name self.module_build_tag = "%s-build" % tag_name From ecd2f5ce46fc4ce9a93dcc3309c8bed0ebeb9247 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 20:23:39 -0400 Subject: [PATCH 033/106] Found another missing config copy statement. This approach may not be scalable. --- rida/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rida/config.py b/rida/config.py index 3386d8cd..bc28c41b 100644 --- a/rida/config.py +++ b/rida/config.py @@ -70,6 +70,7 @@ def from_file(filename=None): conf.pdc_develop = default.get("pdc_develop") conf.koji_config = default.get("koji_config") conf.koji_profile = default.get("koji_profile") + conf.koji_arches = json.loads(default.get("koji_arches")) conf.scmurls = json.loads(default.get("scmurls")) conf.rpms_default_repository = default.get("rpms_default_repository") conf.rpms_allow_repository = asbool(default.get("rpms_allow_repository")) From e6fbde4837575106f0a232e9ce84ceafaa2fd54c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 22:46:20 -0400 Subject: [PATCH 034/106] Improved logging in KojiModuleBuilder. --- rida/builder.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 5cac5af2..7a1cf928 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -216,6 +216,7 @@ class KojiModuleBuilder(GenericBuilder): self.module_build_tag = chkbuildtag self.module_target = chktarget self.__prep = True + log.info("%r buildroot resumed." % self) def buildroot_prep(self): """ @@ -231,13 +232,16 @@ class KojiModuleBuilder(GenericBuilder): self.module_target = self._koji_add_target(self.module_target, self.module_build_tag, self.module_tag) self.__prep = True + log.info("%r buildroot prepared." % self) def buildroot_add_dependency(self, dependencies): tags = [self._get_tag(d)['name'] for d in dependencies] + log.info("%r adding deps for %r" % (self, tags)) self._koji_add_many_tag_inheritance(self.module_build_tag, tags) def buildroot_add_artifacts(self, artifacts): # TODO: import /usr/bin/koji's TaskWatcher() + log.info("%r adding artifacts %r" % (self, artifacts)) for nvr in artifacts: self.koji_session.tagBuild(self.module_build_tag, nvr, force=True) @@ -246,7 +250,7 @@ class KojiModuleBuilder(GenericBuilder): cmd = "koji -p %s wait-repo %s " % (self._koji_profile_name, self.module_build_tag['name']) if artifact: cmd += " --build %s" % artifact - print ("Waiting for buildroot(%s) to be ready" % (self.module_build_tag['name'])) + log.info("Waiting for buildroot(%s) to be ready" % (self.module_build_tag['name'])) run(cmd) # wait till repo is current def build(self, artifact_name, source): @@ -261,7 +265,8 @@ class KojiModuleBuilder(GenericBuilder): raise NotImplementedError("Only scm url is currently supported, got source='%s'" % source) self._koji_whitelist_packages([artifact_name,]) build_id = self.koji_session.build(source, self.module_target['name']) - print("Building %s (build_id=%s)." % (source, build_id)) + log.info("%r submitted build of %s (build_id=%s)" % ( + self, source, build_id)) return build_id def _get_tag(self, tag, strict=True): @@ -302,12 +307,15 @@ class KojiModuleBuilder(GenericBuilder): raise ValueError("Expected dict {'group' : [str(package1), ...]") dest_tag = self._get_tag(dest_tag)['name'] - groups = dict([(p['name'], p['group_id']) for p in self.koji_session.getTagGroups(dest_tag, inherit=False)]) + groups = dict([ + (p['name'], p['group_id']) + for p in self.koji_session.getTagGroups(dest_tag, inherit=False) + ]) for group, packages in groups.iteritems(): group_id = groups.get(group, None) if group_id is not None: - print("Group %s already exists for tag %s" % (group, dest_tag)) - return 1 + log.warning("Group %s already exists for tag %s" % (group, dest_tag)) + continue self.koji_session.groupListAdd(dest_tag, group) for pkg in packages: self.koji_session.groupPackageListAdd(dest_tag, group, pkg) @@ -344,7 +352,7 @@ class KojiModuleBuilder(GenericBuilder): for package in packages: package_id = pkglist.get(package, None) if not package_id is None: - print ("Package %s already exists in tag %s" % (package, self.module_tag['name'])) + log.warn("Package %s already exists in tag %s" % (package, self.module_tag['name'])) continue to_add.append(package) From 61a1e8d89f05d2b6f32c3ec3e29b02f458220b9f Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 22:47:22 -0400 Subject: [PATCH 035/106] Return None here instead of raising an exception. --- rida/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/database.py b/rida/database.py index 501c0c2f..6805ed3d 100644 --- a/rida/database.py +++ b/rida/database.py @@ -147,7 +147,7 @@ class ModuleBuild(Base): def from_fedmsg(cls, session, msg): if '.module.' not in msg['topic']: raise ValueError("%r is not a module message." % msg['topic']) - return session.query(cls).filter(cls.id==msg['msg']['id']).one() + return session.query(cls).filter(cls.id==msg['msg']['id']).first() @classmethod def create(cls, session, conf, name, version, release, modulemd): @@ -243,7 +243,7 @@ class ComponentBuild(Base): def from_fedmsg(cls, session, msg): if '.buildsys.build.state.change' not in msg['topic']: raise ValueError("%r is not a koji message." % msg['topic']) - return session.query(cls).filter(cls.build_id==msg['msg']['build_id']).one() + return session.query(cls).filter(cls.build_id==msg['msg']['build_id']).first() def json(self): return { From 2d33ee16029d1cc3bacf7919a2337058b64db180 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 23:09:15 -0400 Subject: [PATCH 036/106] Simplify this. The code is equivalent. --- rida/database.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/rida/database.py b/rida/database.py index 6805ed3d..9053eb8d 100644 --- a/rida/database.py +++ b/rida/database.py @@ -199,12 +199,8 @@ class ModuleBuild(Base): count = query.count() if count > 1: raise RuntimeError("%r module builds in flight for %r" % (count, koji_tag)) - elif count == 0: - # No builds in flight scheduled by us. Just ignore this. - return None - # Otherwise, there is exactly one module build - it must be ours. - return query.one() + return query.first() def json(self): return { From 7dc2d1ad4252abffde009932e51c2a2d422667b1 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 23:09:48 -0400 Subject: [PATCH 037/106] Bugfix. The attribute is koji_tag. --- rida/scheduler/handlers/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index d5479f39..d2259dc6 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -52,7 +52,7 @@ def wait(config, session, msg): # associated with which koji tag, so that when their repos are regenerated # in koji we can figure out which for which module build that event is # relevant. - build.tag = tag + build.koji_tag = tag dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) From e879bc7f814a5682a3b930cdf754b7ca007ff913 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 23:10:17 -0400 Subject: [PATCH 038/106] Key off the build tag here... .. and we want to map it back to the main tag. --- rida/scheduler/handlers/repos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index e6af1d0e..052dbb04 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -36,7 +36,7 @@ def done(config, session, msg): """ Called whenever koji rebuilds a repo, any repo. """ # First, find our ModuleBuild associated with this repo, if any. - tag = msg['msg']['tag'] + tag = msg['msg']['tag'].strip('-build') module_build = rida.database.ModuleBuild.get_active_by_koji_tag( session, koji_tag=tag) if not module_build: From 3c8e8a4b0215a739c5214f98ba872de5f1ebb3a7 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 23:10:51 -0400 Subject: [PATCH 039/106] This log message is quite useful. --- rida/scheduler/handlers/repos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 052dbb04..cdca4def 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -40,7 +40,7 @@ def done(config, session, msg): module_build = rida.database.ModuleBuild.get_active_by_koji_tag( session, koji_tag=tag) if not module_build: - log.debug("No module build found associated with koji tag %r" % tag) + log.info("No module build found associated with koji tag %r" % tag) return unbuilt_components = ( From 79a1aff19691541910e660539bcb0044ae6ba312 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 23:13:01 -0400 Subject: [PATCH 040/106] Typofix. --- rida/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/builder.py b/rida/builder.py index 7a1cf928..e0629201 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -209,7 +209,7 @@ class KojiModuleBuilder(GenericBuilder): chkbuildtag = self.koji_session.getTag(self.module_build_tag) if not chkbuildtag: raise SystemError("Build Tag %s doesn't exist" % self.module_build_tag) - chktarget = self.koji_session.getBuildTarget(self.module_build_tag) + chktarget = self.koji_session.getBuildTarget(self.module_target) if not chktarget: raise SystemError("Target %s doesn't exist" % self.module_target) self.module_tag = chktag From 67fa46203d1bec446b9abca2a79e18673571e206 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 23:18:01 -0400 Subject: [PATCH 041/106] Clear out some spammy fields when logging the message body. --- rida/scheduler/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index e64e4b47..bcfce364 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -118,6 +118,10 @@ class Messaging(threading.Thread): self.process_message(msg) except Exception: log.exception("Failed while handling %r" % msg['msg_id']) + # Log the body of the message too, but clear out some spammy + # fields that are of no use to a human reader. + msg.pop('certificate', None) + msg.pop('signature', None) log.info(pprint.pformat(msg)) def process_message(self, msg): From 4cc3e1271a8a08d27e73d814bfe4fb0b09e2415c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 16 Jul 2016 23:19:54 -0400 Subject: [PATCH 042/106] Typofix. --- rida/scheduler/handlers/repos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index cdca4def..60d75e72 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -53,7 +53,7 @@ def done(config, session, msg): for component_build in unbuilt_components: scmurl = "{dist_git}/rpms/{package}?#{gitref}".format( - dist_git=config.dist_git_url, + dist_git=config.rpms_default_repository, package=component_build.package, gitref=component_build.gitref, # This is the update stream ) From 7d77463124e1769aecf5b57610ff3a3a771c5e10 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 17 Jul 2016 00:10:23 -0400 Subject: [PATCH 043/106] Get the gitref from rida.py to ridad.py. --- rida.py | 7 ++++++- rida/database.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rida.py b/rida.py index cb6c84e5..eac87544 100755 --- a/rida.py +++ b/rida.py @@ -135,7 +135,12 @@ def submit_build(): return failure("Failed to get the latest commit: %s" % pkgname, 422) if not rida.scm.SCM(pkg["repository"] + "?#" + pkg["commit"]).is_available(): return failure("Cannot checkout %s" % pkgname, 422) - build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") + build = rida.database.ComponentBuild( + module_id=module.id, + package=pkgname, + format="rpms", + gitref=pkg["commit"], # TODO - re-evaluate this w.r.t. supported branches + ) db.session.add(build) module.modulemd = mmd.dumps() module.transition(conf, rida.database.BUILD_STATES["wait"]) diff --git a/rida/database.py b/rida/database.py index 9053eb8d..0a39ce80 100644 --- a/rida/database.py +++ b/rida/database.py @@ -226,6 +226,7 @@ class ComponentBuild(Base): __tablename__ = "component_builds" id = Column(Integer, primary_key=True) package = Column(String, nullable=False) + gitref = Column(String, nullable=False) # XXX: Consider making this a proper ENUM format = Column(String, nullable=False) build_id = Column(Integer) # This is the id of the build in koji From 45bc4c55acb9b025efe9acf9473fd6b33228061c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 17 Jul 2016 00:11:00 -0400 Subject: [PATCH 044/106] Some summary logging from ridad. --- rida/builder.py | 4 ++++ rida/database.py | 10 +++++++--- rida/scheduler/main.py | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index e0629201..791ab165 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -199,6 +199,10 @@ class KojiModuleBuilder(GenericBuilder): self.module_build_tag = "%s-build" % tag_name self.module_target = tag_name + def __repr__(self): + return "" % ( + self.module_str, self.module_tag) + def buildroot_resume(self): # XXX: experimental """ Resume existing buildroot. Sets __prep=True diff --git a/rida/database.py b/rida/database.py index 0a39ce80..12c71652 100644 --- a/rida/database.py +++ b/rida/database.py @@ -70,6 +70,8 @@ BUILD_STATES = { "ready": 5, } +INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()} + class RidaBase(object): # TODO -- we can implement functionality here common to all our model @@ -218,8 +220,9 @@ class ModuleBuild(Base): } def __repr__(self): - return "" % ( - self.name, self.version, self.release) + return "" % ( + self.name, self.version, self.release, + INVERSE_BUILD_STATES[self.state]) class ComponentBuild(Base): @@ -253,4 +256,5 @@ class ComponentBuild(Base): } def __repr__(self): - return "" % (self.package, self.module_id) + return "" % ( + self.package, self.module_id, self.state) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index bcfce364..9cf54afd 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -32,6 +32,7 @@ proper scheduling component builds in the supported build systems. import inspect import logging +import operator import os import pprint import threading @@ -146,6 +147,8 @@ class Messaging(threading.Thread): class Polling(threading.Thread): def run(self): while True: + with rida.database.Database(config) as session: + self.log_summary(session) with rida.database.Database(config) as session: self.process_waiting_module_builds(session) with rida.database.Database(config) as session: @@ -155,6 +158,21 @@ class Polling(threading.Thread): log.info("Polling thread sleeping, %rs" % config.polling_interval) time.sleep(config.polling_interval) + def log_summary(self, session): + log.info("Current status:") + states = sorted(rida.BUILD_STATES.items(), key=operator.itemgetter(1)) + for name, code in states: + query = session.query(rida.database.ModuleBuild) + count = query.filter_by(state=code).count() + if count: + log.info(" * %i module builds in the %s state." % (count, name)) + if name == 'build': + for module_build in query.all(): + log.info(" * %r" % module_build) + for component_build in module_build.component_builds: + log.info(" * %r" % component_build) + + def process_waiting_module_builds(self, session): log.info("Looking for module builds stuck in the wait state.") builds = rida.database.ModuleBuild.by_state(session, "wait") From f32201798289f2630463532a3e6a0ae3ad7cc0ff Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 17 Jul 2016 00:11:13 -0400 Subject: [PATCH 045/106] Koji build states are actually integers. --- rida/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index 12c71652..be24c26c 100644 --- a/rida/database.py +++ b/rida/database.py @@ -234,7 +234,7 @@ class ComponentBuild(Base): format = Column(String, nullable=False) build_id = Column(Integer) # This is the id of the build in koji # XXX: Consider making this a proper ENUM (or an int) - state = Column(String) + state = Column(Integer) module_id = Column(Integer, ForeignKey('module_builds.id'), nullable=False) module_build = relationship('ModuleBuild', backref='component_builds', lazy=False) From 9190a48b758775264491d963c78f348ecb28ea05 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 17 Jul 2016 00:13:59 -0400 Subject: [PATCH 046/106] A little less confusing. --- rida/builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 791ab165..9f76a2bd 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -156,10 +156,10 @@ class KojiModuleBuilder(GenericBuilder): self.__prep = False self._koji_profile_name = config.koji_profile log.debug("Using koji profile %r" % self._koji_profile_name) - self.koji_module = koji.get_profile_module(self._koji_profile_name) + self.koji_profile = koji.get_profile_module(self._koji_profile_name) opts = {} - koji_config = self.koji_module.config + koji_config = self.koji_profile.config krbservice = getattr(koji_config, "krbservice", None) if krbservice: From 379fc3fe5a330cf86c9ccd1d1a40d1fea2afc093 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 17 Jul 2016 00:16:57 -0400 Subject: [PATCH 047/106] Get the test suite running again. --- tests/test_scheduler/test_repo_done.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py index d010ab0a..09e8db65 100644 --- a/tests/test_scheduler/test_repo_done.py +++ b/tests/test_scheduler/test_repo_done.py @@ -30,7 +30,8 @@ class TestRepoDone(unittest.TestCase): def setUp(self): self.config = mock.Mock() - self.config.dist_git_url = 'dist_git_url' + self.config.rpms_default_repository = 'dist_git_url' + self.config.koji_profile = 'staging' # TODO - point at a fake test config self.session = mock.Mock() self.fn = rida.scheduler.handlers.repos.done From 82bab660c8d4c48535437b3940b875f1d621ec28 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Jul 2016 10:45:40 -0400 Subject: [PATCH 048/106] Be honest. This is a task_id, not a build_id. --- rida/builder.py | 10 +++++----- rida/database.py | 6 +++--- rida/logger.py | 2 +- rida/scheduler/handlers/repos.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 9f76a2bd..0fcde89c 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -260,7 +260,7 @@ class KojiModuleBuilder(GenericBuilder): def build(self, artifact_name, source): """ :param source : scmurl to spec repository - :return koji build id + :return koji build task id """ if not self.__prep: raise RuntimeError("Buildroot is not prep-ed") @@ -268,10 +268,10 @@ class KojiModuleBuilder(GenericBuilder): if '://' not in source: raise NotImplementedError("Only scm url is currently supported, got source='%s'" % source) self._koji_whitelist_packages([artifact_name,]) - build_id = self.koji_session.build(source, self.module_target['name']) - log.info("%r submitted build of %s (build_id=%s)" % ( - self, source, build_id)) - return build_id + task_id = self.koji_session.build(source, self.module_target['name']) + log.info("%r submitted build of %s (task_id=%s)" % ( + self, source, task_id)) + return task_id def _get_tag(self, tag, strict=True): if isinstance(tag, dict): diff --git a/rida/database.py b/rida/database.py index be24c26c..9f4a2997 100644 --- a/rida/database.py +++ b/rida/database.py @@ -232,7 +232,7 @@ class ComponentBuild(Base): gitref = Column(String, nullable=False) # XXX: Consider making this a proper ENUM format = Column(String, nullable=False) - build_id = Column(Integer) # This is the id of the build in koji + task_id = Column(Integer) # This is the id of the build in koji # XXX: Consider making this a proper ENUM (or an int) state = Column(Integer) @@ -243,14 +243,14 @@ class ComponentBuild(Base): def from_fedmsg(cls, session, msg): if '.buildsys.build.state.change' not in msg['topic']: raise ValueError("%r is not a koji message." % msg['topic']) - return session.query(cls).filter(cls.build_id==msg['msg']['build_id']).first() + return session.query(cls).filter(cls.task_id==msg['msg']['task_id']).first() def json(self): return { 'id': self.id, 'package': self.package, 'format': self.format, - 'build_id': self.build_id, + 'task_id': self.task_id, 'state': self.state, 'module_build': self.module_id, } diff --git a/rida/logger.py b/rida/logger.py index b1596672..eb48cf8f 100644 --- a/rida/logger.py +++ b/rida/logger.py @@ -38,7 +38,7 @@ import logging logging.debug("Phasers are set to stun.") logging.info("%s tried to build something", username) -logging.warn("%s failed to build", build_id) +logging.warn("%s failed to build", task_id) """ diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 60d75e72..76b7c8bb 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -59,5 +59,5 @@ def done(config, session, msg): ) artifact_name = 'TODO' component_build.state = koji.BUILD_STATES['BUILDING'] - component_build.build_id = builder.build(artifact_name, scmurl) + component_build.task_id = builder.build(artifact_name, scmurl) session.commit() From f85665f42f77274764878d1160158fb841c68feb Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Jul 2016 10:46:33 -0400 Subject: [PATCH 049/106] Skip the query to PDC for the koji-tag for now. --- rida/pdc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rida/pdc.py b/rida/pdc.py index 58c6dff3..4fcd238d 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -140,7 +140,11 @@ def get_module_tag(session, module_info, strict=False): :param module_info: list of module_info dicts :return: koji tag string """ - return get_module(session, module_info, strict=strict)['koji_tag'] + # TODO -- get this from PDC some day... for now, we're just going to + # construct the module tag name from the module attrs we already know + # about. + #return get_module(session, module_info, strict=strict)['koji_tag'] + return "{name}-{version}-{release}".format(**module_info) def module_depsolving_wrapper(session, module_list, strict=False): """ From 1a66202afdf03acdf01360e5b4b38a45d55510e1 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Mon, 18 Jul 2016 19:17:58 +0200 Subject: [PATCH 050/106] read profile from supplied file --- rida/builder.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 0fcde89c..54779136 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -32,6 +32,7 @@ from abc import ABCMeta, abstractmethod import logging import os +from optparse import OptionParser from kobo.shortcuts import run import koji @@ -143,6 +144,14 @@ class Builder: else: raise ValueError("Builder backend='%s' not recognized" % backend) +def _get_opts_from_dict(data): + """koji requires config in optparse opts style""" + config = OptionParser() + opts, _ = config.parse_args() + for key, value in data.iteritems(): + setattr(opts, key, value) + return opts + class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ @@ -156,18 +165,18 @@ class KojiModuleBuilder(GenericBuilder): self.__prep = False self._koji_profile_name = config.koji_profile log.debug("Using koji profile %r" % self._koji_profile_name) - self.koji_profile = koji.get_profile_module(self._koji_profile_name) - opts = {} + log.debug ("Using koji_config: %s" % config.koji_config) - koji_config = self.koji_profile.config + koji_config = _get_opts_from_dict(koji.read_config(profile_name=config.koji_profile, user_config=config.koji_config)) + self.koji_profile = koji.get_profile_module(self._koji_profile_name, config=koji_config) krbservice = getattr(koji_config, "krbservice", None) if krbservice: - opts["krbservice"] = krbservice + koji_config.krbservice = krbservice address = koji_config.server - log.info("Connecting to koji %r, %r" % (address, opts)) - self.koji_session = koji.ClientSession(address, opts=opts) + log.info("Connecting to koji %r, %r" % (address, koji_config)) + self.koji_session = koji.ClientSession(address, opts=vars(koji_config)) authtype = koji_config.authtype if authtype == "kerberos": @@ -326,6 +335,7 @@ class KojiModuleBuilder(GenericBuilder): def _koji_create_tag(self, tag_name, arches=None, fail_if_exists=True, perm=None): + print ("Creating tag %s" % tag_name) chktag = self.koji_session.getTag(tag_name) if chktag and fail_if_exists: raise SystemError("Tag %s already exist" % tag_name) From c0408b148b9719fb13e864d09b049c76cfefd260 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Mon, 18 Jul 2016 22:29:02 +0200 Subject: [PATCH 051/106] builder: remove wait_buildroot and turn ClientSession into staticmethod Signed-off-by: Lubos Kocman --- rida/builder.py | 94 ++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 52 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 54779136..7085aafd 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -113,14 +113,6 @@ class GenericBuilder: """ raise NotImplementedError() - @abstractmethod - def buildroot_ready(self, artifact=None): - """ - :param artifact=None: wait for specific artifact to be present - waits for buildroot to be ready and contain given artifact - """ - raise NotImplementedError() - @abstractmethod def build(self, artifact_name, source): """ @@ -159,47 +151,16 @@ class KojiModuleBuilder(GenericBuilder): def __init__(self, module, config, tag_name): """ - :param koji_profile: koji profile to be used + :param module: string representing module + :param config: rida.config.Config instance + :param tag_name: name of tag for given module """ self.module_str = module self.__prep = False - self._koji_profile_name = config.koji_profile - log.debug("Using koji profile %r" % self._koji_profile_name) + log.debug("Using koji profile %r" % config.koji_profile) log.debug ("Using koji_config: %s" % config.koji_config) - koji_config = _get_opts_from_dict(koji.read_config(profile_name=config.koji_profile, user_config=config.koji_config)) - self.koji_profile = koji.get_profile_module(self._koji_profile_name, config=koji_config) - - krbservice = getattr(koji_config, "krbservice", None) - if krbservice: - koji_config.krbservice = krbservice - - address = koji_config.server - log.info("Connecting to koji %r, %r" % (address, koji_config)) - self.koji_session = koji.ClientSession(address, opts=vars(koji_config)) - - authtype = koji_config.authtype - if authtype == "kerberos": - keytab = getattr(koji_config, "keytab", None) - principal = getattr(koji_config, "principal", None) - if keytab and principal: - self.koji_session.krb_login( - principal=principal, - keytab=keytab, - proxyuser=None, - ) - else: - self.koji_session.krb_login() - elif authtype == "ssl": - self.koji_session.ssl_login( - os.path.expanduser(koji_config.cert), - None, - os.path.expanduser(koji_config.serverca), - proxyuser=None, - ) - else: - raise ValueError("Unrecognized koji authtype %r" % authtype) - + self.koji_session, self.koji_module = self.get_session_from_config(config) self.arches = config.koji_arches if not self.arches: raise ValueError("No koji_arches specified in the config.") @@ -212,6 +173,43 @@ class KojiModuleBuilder(GenericBuilder): return "" % ( self.module_str, self.module_tag) + + @staticmethod + def get_session_from_config(config): + koji_config = _get_opts_from_dict(koji.read_config(profile_name=config.koji_profile, user_config=config.koji_config)) + koji_module = koji.get_profile_module(config.koji_profile, config=koji_config) + + krbservice = getattr(koji_config, "krbservice", None) + if krbservice: + koji_config.krbservice = krbservice + + address = koji_config.server + log.info("Connecting to koji %r, %r" % (address, koji_config)) + koji_session = koji.ClientSession(address, opts=vars(koji_config)) + + authtype = koji_config.authtype + if authtype == "kerberos": + keytab = getattr(koji_config, "keytab", None) + principal = getattr(koji_config, "principal", None) + if keytab and principal: + koji_session.krb_login( + principal=principal, + keytab=keytab, + proxyuser=None, + ) + else: + koji_session.krb_login() + elif authtype == "ssl": + koji_session.ssl_login( + os.path.expanduser(koji_config.cert), + None, + os.path.expanduser(koji_config.serverca), + proxyuser=None, + ) + else: + raise ValueError("Unrecognized koji authtype %r" % authtype) + return (koji_session, koji_module) + def buildroot_resume(self): # XXX: experimental """ Resume existing buildroot. Sets __prep=True @@ -258,14 +256,6 @@ class KojiModuleBuilder(GenericBuilder): for nvr in artifacts: self.koji_session.tagBuild(self.module_build_tag, nvr, force=True) - def buildroot_ready(self, artifact=None): - # XXX: steal code from /usr/bin/koji - cmd = "koji -p %s wait-repo %s " % (self._koji_profile_name, self.module_build_tag['name']) - if artifact: - cmd += " --build %s" % artifact - log.info("Waiting for buildroot(%s) to be ready" % (self.module_build_tag['name'])) - run(cmd) # wait till repo is current - def build(self, artifact_name, source): """ :param source : scmurl to spec repository From bd28b7852afd34c6e2edf1c7395b2821703a98f5 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Mon, 18 Jul 2016 22:29:59 +0200 Subject: [PATCH 052/106] database: listen to rida.component.state.change (failover) --- rida/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index 9f4a2997..79f2cf4f 100644 --- a/rida/database.py +++ b/rida/database.py @@ -241,7 +241,7 @@ class ComponentBuild(Base): @classmethod def from_fedmsg(cls, session, msg): - if '.buildsys.build.state.change' not in msg['topic']: + if 'component.state.change' not in msg['topic'] and '.buildsys.build.state.change' not in msg['topic']: raise ValueError("%r is not a koji message." % msg['topic']) return session.query(cls).filter(cls.task_id==msg['msg']['task_id']).first() From fe7c9dbec0678f96882749cb91ab51e6f8404a21 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Mon, 18 Jul 2016 22:31:24 +0200 Subject: [PATCH 053/106] ensure that wait() is being called only in wait state * I'm little bit suspicious that something is calling wait() from other states as well * This is to keep track of such calls, so we can identify them later --- rida/scheduler/handlers/modules.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index d2259dc6..e4a59375 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -42,6 +42,11 @@ def wait(config, session, msg): """ build = rida.database.ModuleBuild.from_fedmsg(session, msg) module_info = build.json() + if module_info['state'] != rida.BUILD_STATES["wait"]: + # XXX: not sure why did we get here from state == 2 (building) FIXTHIS + print("Invalid state %s for wait()" % module_info['state']) + log.error("Invalid state %s for wait(). Msg=%s" % (module_info['state'], msg)) + return log.info("Found module_info=%s from message" % module_info) pdc_session = rida.pdc.get_pdc_client_session(config) From ff84228502f9a44ca849e0cde9f5271a3b039af6 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Mon, 18 Jul 2016 22:33:26 +0200 Subject: [PATCH 054/106] Add handler for builds which didn't manage to get builID * this is typically for builds where srpm task failed we don't receive any message on such failures, so I'm rather generating internal one --- rida/scheduler/main.py | 54 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 9cf54afd..3854588e 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -133,6 +133,8 @@ class Messaging(threading.Thread): handler = self.on_repo_change elif '.buildsys.build.state.change' in msg['topic']: handler = self.on_build_change[msg['msg']['new']] + elif 'rida.component.state.change' in msg['topic']: + handler = self.on_build_change[msg['msg']['new']] elif '.rida.module.state.change' in msg['topic']: handler = self.on_module_change[module_build_state_from_msg(msg)] else: @@ -149,15 +151,63 @@ class Polling(threading.Thread): while True: with rida.database.Database(config) as session: self.log_summary(session) - with rida.database.Database(config) as session: - self.process_waiting_module_builds(session) + # XXX: detect whether it's really stucked first + #with rida.database.Database(config) as session: + # self.process_waiting_module_builds(session) with rida.database.Database(config) as session: self.process_open_component_builds(session) with rida.database.Database(config) as session: self.process_lingering_module_builds(session) + with rida.database.Database(config) as session: + self.fail_lost_builds(session) + log.info("Polling thread sleeping, %rs" % config.polling_interval) time.sleep(config.polling_interval) + def fail_lost_builds(self, session): + # This function is supposed to be handling only + # the part which can't be updated trough messaging (srpm-build failures). + # Please keep it fit `n` slim. We do want rest to be processed elsewhere + + # TODO re-use + + if config.system == "koji": + def send_fail_task_msg(component_build, task_info): + log.debug("Failing task=%r" % task_info) + + rida.messaging.publish( + modname='rida', + topic='component.state.change', + msg={ + "method": "build", + "attribute": "state", + "new": koji.BUILD_STATES['FAILED'], + "task_id": component_build.task_id}, + + backend=config.messaging, + ) + + koji_session, _ = rida.builder.KojiModuleBuilder.get_session_from_config(config) + state = koji.BUILD_STATES['BUILDING'] # Check tasks that we track as BUILDING + log.info("Querying tasks for statuses:") + query = session.query(rida.database.ComponentBuild) + res = query.filter(state==koji.BUILD_STATES['BUILDING']).all() + + log.info("Checking status for %d tasks." % len(res)) + for component_build in res: + log.debug(component_build.json()) + if not component_build.task_id: # Don't check tasks which has not been triggered yet + continue + + log.info("Checking status of task_id=%s" % component_build.task_id) + task_info = koji_session.getTaskInfo(component_build.task_id) + + if task_info['state'] in (koji.TASK_STATES['CANCELED'], koji.TASK_STATES['FAILED']): + send_fail_task_msg(component_build, task_info) + + else: + raise NotImplementedError("Buildsystem %r is not supported." % config.system) + def log_summary(self, session): log.info("Current status:") states = sorted(rida.BUILD_STATES.items(), key=operator.itemgetter(1)) From 05980dacff0334c41b7525c8bcbc28283d3f07ff Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 10:25:54 +0200 Subject: [PATCH 055/106] Add buildroot_ready, disttag related function Signed-off-by: Lubos Kocman --- rida/builder.py | 120 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 7085aafd..f163ebc8 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -35,6 +35,12 @@ import os from optparse import OptionParser from kobo.shortcuts import run import koji +import tempfile +import glob +import datetime +import time +import random +import string log = logging.getLogger(__name__) @@ -97,6 +103,14 @@ class GenericBuilder: """ raise NotImplementedError() + @abstractmethod + def buildroot_ready(self, artifacts=None): + """ + :param artifacts=None : a list of artifacts supposed to be in buildroot + return when buildroot is ready (or contain specified artifact) + """ + raise NotImplementedError() + @abstractmethod def buildroot_add_dependency(self, dependencies): """ @@ -173,6 +187,91 @@ class KojiModuleBuilder(GenericBuilder): return "" % ( self.module_str, self.module_tag) + def buildroot_ready(self, artifacts=None): + target_info = self.koji_session.getBuildTarget(self.module_target) + assert target_info, "Invalid build target" + + timeout = 120 # minutes see * 60 + tag_id = target_info['build_tag'] + start = time.time() + last_repo = None + repo = self.koji_session.getRepo(tag_id) + + while True: + if artifacts and repo and repo != last_repo: + if koji.util.checkForBuilds(self.koji_session, tag_id, artifacts, repo['create_event'], latest=True): + return + + if (time.time() - start) >= (timeout * 60.0): + return 1 + + time.sleep(60) + last_repo = repo + repo = self.koji_session.getRepo(tag_id) + + if not artifacts: + if repo != last_repo: + return + + @staticmethod + def get_disttag_srpm(disttag): + + #Taken from Karsten's create-distmacro-pkg.sh + # - however removed any provides to system-release/redhat-release + + name = 'module-build-macros' + version = "0.1" + release = "1" + today = datetime.date.today().strftime('%a %b %d %Y') + + spec_content = """%global dist {disttag} +Name: {name} +Version: {version} +Release: {release}%dist +Summary: Package containing macros required to build generic module +BuildArch: noarch + +Group: System Environment/Base +License: MIT +URL: http://fedoraproject.org + +%description +This package is used for building modules with a different dist tag. +It provides a file /usr/lib/rpm/macros.d/macro.modules and gets read +after macro.dist, thus overwriting macros of macro.dist like %%dist +It should NEVER be installed on any system as it will really mess up + updates, builds, .... + + +%build + +%install +mkdir -p %buildroot/%_rpmconfigdir/macro.d 2>/dev/null |: +echo %%dist %dist > %buildroot/%_rpmconfigdir/macro.modules +chmod 644 %buildroot/%_rpmconfigdir/macro.modules + + +%files +%_rpmconfigdir/macro.modules + + + +%changelog +* {today} Fedora-Modularity - {version}-{release}{disttag} +- autogenerated macro by Rida "The Orchestrator" +""".format(disttag=disttag, today=today, name=name, version=version, release=release) + td = tempfile.mkdtemp(prefix="rida-build-macros") + fd = open(os.path.join(td, "%s.spec" % name), "w") + fd.write(spec_content) + fd.close() + log.debug("Building %s.spec" % name) + ret, out = run('rpmbuild -bs %s.spec --define "_topdir %s"' % (name, td), workdir=td) + sdir = os.path.join(td, "SRPMS") + srpm_paths = glob.glob("%s/*.src.rpm" % sdir) + assert len(srpm_paths) == 1, "Expected exactly 1 srpm in %s. Got %s" % (sdir, srpm_paths) + + log.debug("Wrote srpm into %s" % srpm_paths[0]) + return srpm_paths[0] @staticmethod def get_session_from_config(config): @@ -261,12 +360,29 @@ class KojiModuleBuilder(GenericBuilder): :param source : scmurl to spec repository :return koji build task id """ + # Taken from /usr/bin/koji + def _unique_path(prefix): + """Create a unique path fragment by appending a path component + to prefix. The path component will consist of a string of letter and numbers + that is unlikely to be a duplicate, but is not guaranteed to be unique.""" + # Use time() in the dirname to provide a little more information when + # browsing the filesystem. + # For some reason repr(time.time()) includes 4 or 5 + # more digits of precision than str(time.time()) + return '%s/%r.%s' % (prefix, time.time(), + ''.join([random.choice(string.ascii_letters) for i in range(8)])) + if not self.__prep: raise RuntimeError("Buildroot is not prep-ed") - if '://' not in source: - raise NotImplementedError("Only scm url is currently supported, got source='%s'" % source) self._koji_whitelist_packages([artifact_name,]) + if '://' not in source: + #treat source as an srpm and upload it + serverdir = _unique_path('cli-build') + callback =None + self.koji_session.uploadWrapper(source, serverdir, callback=callback) + source = "%s/%s" % (serverdir, os.path.basename(source)) + task_id = self.koji_session.build(source, self.module_target['name']) log.info("%r submitted build of %s (task_id=%s)" % ( self, source, task_id)) From ce59c608be3fb940e63e37028d9250c60279b08a Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 10:27:13 +0200 Subject: [PATCH 056/106] modules.py: create modules-macro inject it to buildroot and wait before building state --- rida/scheduler/handlers/modules.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index e4a59375..91a4e078 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -31,6 +31,12 @@ import logging log = logging.getLogger(__name__) +def get_rpm_release_from_tag(tag): + return tag.replace("-", "_") + +def get_artifact_from_srpm(srpm_path): + return os.path.basename(srpm_path).replace(".src.rpm", "") + def wait(config, session, msg): """ Called whenever a module enters the 'wait' state. @@ -62,10 +68,12 @@ def wait(config, session, msg): dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) build.buildroot_task_id = builder.buildroot_prep() + buildroot_add_dependency(["f24-build",]) # XXX: hack remove once we have dependencies builder.buildroot_add_dependency(dependencies) - # TODO: build srpm with dist_tag macros - # TODO submit build from srpm to koji - # TODO: buildroot.add_artifact(build_with_dist_tags) - # TODO: buildroot.ready(artifact=$artifact) + srpm = builder.get_disttag_srpm(disttag="%s" % get_rpm_release_from_tag(tag)) + builder.build(srpm) + artifact = get_artifact_from_srpm(srpm) + bulder.add_artifact(artifact) + builder.buildroot_ready(artifacts=[artifact,]) build.transition(config, state="build") # Wait for the buildroot to be ready. session.commit() From 033a0da5ba94841240f2026ac52dd2dd3ac21a31 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 10:15:28 -0400 Subject: [PATCH 057/106] Split this up into three threads so we can spoof ourselves. --- rida/scheduler/main.py | 91 +++++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 3854588e..fd8a8924 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -38,6 +38,14 @@ import pprint import threading import time +try: + # Py3 + import queue +except ImportError: + # Py2 + import Queue as queue + + import rida.config import rida.logger import rida.messaging @@ -61,16 +69,34 @@ else: config = rida.config.from_file() +class STOP_WORK(object): + """ A sentinel value, indicating that work should be stopped. """ + pass + + def module_build_state_from_msg(msg): state = int(msg['msg']['state']) # TODO better handling assert state in rida.BUILD_STATES.values(), "state=%s(%s) is not in %s" % (state, type(state), rida.BUILD_STATES.values()) return state -class Messaging(threading.Thread): - def __init__(self, *args, **kwargs): - super(Messaging, self).__init__(*args, **kwargs) +class MessageIngest(threading.Thread): + def __init__(self, outgoing_work_queue, *args, **kwargs): + self.outgoing_work_queue = outgoing_work_queue + super(MessageIngest, self).__init__(*args, **kwargs) + + + def run(self): + for msg in rida.messaging.listen(backend=config.messaging): + self.outgoing_work_queue.put(msg) + + +class MessageWorker(threading.Thread): + + def __init__(self, incoming_work_queue, *args, **kwargs): + self.incoming_work_queue = incoming_work_queue + super(MessageWorker, self).__init__(*args, **kwargs) # These are our main lookup tables for figuring out what to run in response # to what messaging events. @@ -114,7 +140,13 @@ class Messaging(threading.Thread): def run(self): self.sanity_check() - for msg in rida.messaging.listen(backend=config.messaging): + while True: + msg = self.incoming_work_queue.get() + + if msg is STOP_WORK: + log.info("Worker thread received STOP_WORK, shutting down...") + break + try: self.process_message(msg) except Exception: @@ -146,7 +178,11 @@ class Messaging(threading.Thread): log.info(" %r: %s, %s" % (handler, msg['topic'], msg['msg_id'])) handler(config, session, msg) -class Polling(threading.Thread): +class Poller(threading.Thread): + def __init__(self, outgoing_work_queue, *args, **kwargs): + self.outgoing_work_queue = outgoing_work_queue + super(Poller, self).__init__(*args, **kwargs) + def run(self): while True: with rida.database.Database(config) as session: @@ -172,21 +208,6 @@ class Polling(threading.Thread): # TODO re-use if config.system == "koji": - def send_fail_task_msg(component_build, task_info): - log.debug("Failing task=%r" % task_info) - - rida.messaging.publish( - modname='rida', - topic='component.state.change', - msg={ - "method": "build", - "attribute": "state", - "new": koji.BUILD_STATES['FAILED'], - "task_id": component_build.task_id}, - - backend=config.messaging, - ) - koji_session, _ = rida.builder.KojiModuleBuilder.get_session_from_config(config) state = koji.BUILD_STATES['BUILDING'] # Check tasks that we track as BUILDING log.info("Querying tasks for statuses:") @@ -202,14 +223,27 @@ class Polling(threading.Thread): log.info("Checking status of task_id=%s" % component_build.task_id) task_info = koji_session.getTaskInfo(component_build.task_id) - if task_info['state'] in (koji.TASK_STATES['CANCELED'], koji.TASK_STATES['FAILED']): - send_fail_task_msg(component_build, task_info) + dead_states = ( + koji.TASK_STATES['CANCELED'], + koji.TASK_STATES['FAILED'], + ) + if task_info['state'] in dead_states: + # Fake a fedmsg message on our internal queue + self.outgoing_work_queue.put({ + 'topic': 'org.fedoraproject.prod.buildsys.build.state.change', + 'msg': { + 'task_id': component_build.task_id, + 'new': koji.BUILD_STATES['FAILED'], + }, + }) else: raise NotImplementedError("Buildsystem %r is not supported." % config.system) def log_summary(self, session): log.info("Current status:") + backlog = self.outgoing_work_queue.qsize() + log.info(" * internal queue backlog is %i." % backlog) states = sorted(rida.BUILD_STATES.items(), key=operator.itemgetter(1)) for name, code in states: query = session.query(rida.database.ModuleBuild) @@ -247,10 +281,19 @@ def main(): rida.logger.init_logging(config) log.info("Starting ridad.") try: - messaging_thread = Messaging() - polling_thread = Polling() + work_queue = queue.Queue() + + # This ingest thread puts work on the queue + messaging_thread = MessageIngest(work_queue) + # This poller does other work, but also sometimes puts work in queue. + polling_thread = Poller(work_queue) + # This worker takes work off the queue and handles it. + worker_thread = MessageWorker(work_queue) + messaging_thread.start() polling_thread.start() + worker_thread.start() + except KeyboardInterrupt: # FIXME: Make this less brutal os._exit() From 1fdabfd8cf4769537498b50337830d0193392d36 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 16:37:09 +0200 Subject: [PATCH 058/106] pdc: make sure that we pass the right filters to pdc --- rida/pdc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rida/pdc.py b/rida/pdc.py index 4fcd238d..d2dc907d 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -121,8 +121,10 @@ def get_module(session, module_info, strict=False): """ module_info = get_variant_dict(module_info) - - retval = session['unreleasedvariants'](page_size=-1, **module_info) + retval = session['unreleasedvariants'](page_size=-1, + variant_name=module_info['variant_name'], + variant_version=module_info['variant_version'], + variant_release=module_info['variant_release']) assert len(retval) <= 1 # Error handling From fcc47d2e3c1b9171c9e6df1090b604b7c7f771d9 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 16:46:29 +0200 Subject: [PATCH 059/106] add cleanup-koji-stg.sh script --- cleanup-koji-stg.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 cleanup-koji-stg.sh diff --git a/cleanup-koji-stg.sh b/cleanup-koji-stg.sh new file mode 100755 index 00000000..306869ae --- /dev/null +++ b/cleanup-koji-stg.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# default's *42* lkocman's *43 as well + +for mvr in testmodule-4.3.43-1 testmodule-4.3.42-1; do + koji --config /etc/rida/koji.conf remove-target $mvr + koji --config /etc/rida/koji.conf remove-tag $mvr + koji --config /etc/rida/koji.conf remove-tag $mvr-build +done From 82ad4e7e9c9dfcd170fa0ecff6ddf16191c23ca0 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 16:47:17 +0200 Subject: [PATCH 060/106] update default pdc server in rida.conf --- rida.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida.conf b/rida.conf index aaeda2c3..d1ea9a31 100644 --- a/rida.conf +++ b/rida.conf @@ -6,7 +6,7 @@ koji_config = ~/.koji/config koji_profile = staging koji_arches = ["x86_64"] db = sqlite:///rida.db -pdc_url = http://fed-mod.org:8000/rest_api/v1 +pdc_url = http://modularity.fedorainfracloud.org:8080/rest_api/v1 pdc_insecure = True pdc_develop = True scmurls = ["git://pkgs.stg.fedoraproject.org/modules/"] From 240d494c93b03d4e0d276495fbf957d29d27ac38 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 17:52:07 +0200 Subject: [PATCH 061/106] builder: fix tagBuild call, add wait_task --- rida/builder.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index f163ebc8..ccaa389a 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -130,7 +130,9 @@ class GenericBuilder: @abstractmethod def build(self, artifact_name, source): """ - :param artifact_name : name of what are we building (used for whitelists) + :param artifact_name : a crucial, since we can't guess a valid srpm name + without having the exact buildroot (collections/macros) + used e.g. for whitelisting packages :param source : a scmurl to repository with receipt (e.g. spec) """ raise NotImplementedError() @@ -353,7 +355,16 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules # TODO: import /usr/bin/koji's TaskWatcher() log.info("%r adding artifacts %r" % (self, artifacts)) for nvr in artifacts: - self.koji_session.tagBuild(self.module_build_tag, nvr, force=True) + # we do need taginfo dict not the string _get_tag() + self.koji_session.tagBuild(self._get_tag(self.module_build_tag), nvr, force=True) + + def wait_task(self, task_id): + """ + :param task_id + :return - task result object + """ + log.info("Waiting for task_id=%s" % task_id) + return self.koji_session.getTaskResult(task_id) def build(self, artifact_name, source): """ From a503031698ea090a04f10e88449a12bc2bbfe687 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 17:52:28 +0200 Subject: [PATCH 062/106] pdc: improve desolving --- rida/pdc.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/rida/pdc.py b/rida/pdc.py index d2dc907d..04539335 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -88,7 +88,7 @@ def get_variant_dict(data): result['variant_release'] = '0' elif is_module_dict(data): - result = {'variant_name': data['name'], 'variant_version': data['version']} + result = {'variant_name': data['name'], 'variant_version': data['version'], 'variant_release': data['release']} if not result: raise ValueError("Couldn't get variant_dict from %s" % data) @@ -107,8 +107,12 @@ def variant_dict_from_str(module_str): module_info = {} - module_info['variant_name'] = module_str[:module_str.find('-')] - module_info['variant_version'] = module_str[module_str.find('-')+1:] + + release_start = module_str.rfind('-') + version_start = module_str.rfind('-', 0, release_start) + module_info['variant_release'] = module_str[release_start+1:] + module_info['variant_version'] = module_str[version_start+1:release_start] + module_info['variant_name'] = module_str[:version_start] module_info['variant_type'] = 'module' return module_info @@ -146,9 +150,10 @@ def get_module_tag(session, module_info, strict=False): # construct the module tag name from the module attrs we already know # about. #return get_module(session, module_info, strict=strict)['koji_tag'] - return "{name}-{version}-{release}".format(**module_info) + variant_data = get_variant_dict(module_info) + return "{variant_name}-{variant_version}-{variant_release}".format(**variant_data) -def module_depsolving_wrapper(session, module_list, strict=False): +def module_depsolving_wrapper(session, module_list, strict=True): """ :param session : PDCClient instance :param module_list: list of module_info dicts @@ -157,14 +162,29 @@ def module_depsolving_wrapper(session, module_list, strict=False): # TODO: implement this # Make sure that these are dicts from PDC ... ensures all values - module_infos = [get_module(session, module, strict=strict) for module in module_list] + module_list = set([get_module_tag(session, x, strict) for x in module_list]) + seen = set() # don't query pdc for the same items all over again - return module_infos + while True: + if seen == module_list: + break -def get_module_dependencies(session, module_info, strict=False): + for module in module_list: + if module in seen: + continue + info = get_module(session, module, strict) + assert info, "Module '%s' not found in PDC" % module + module_list.update([x['dependency'] for x in info['build_deps']]) + seen.add(module) + module_list.update(info['build_deps']) + + return list(module_list) + +def get_module_runtime_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance :param module_infos : a dict containing filters for pdc + :param strict=False : don't raise exception if None is returned Example minimal module_info {'variant_name': module_name, 'variant_version': module_version, 'variant_type': 'module'} """ @@ -172,7 +192,7 @@ def get_module_dependencies(session, module_info, strict=False): deps = [] module_info = get_module(session, module_info, strict=strict) - if module_info and module_info.get('runtime_deps'): + if module_info and module_info.get('runtime_deps', None): deps = [x['dependency'] for x in module_info['runtime_deps']] deps = module_depsolving_wrapper(session, deps, strict=strict) @@ -190,7 +210,7 @@ def get_module_build_dependencies(session, module_info, strict=False): deps = [] module_info = get_module(session, module_info, strict=strict) - if module_info and module_info.get('build_deps'): + if module_info and module_info.get('build_deps', None): deps = [x['dependency'] for x in module_info['build_deps']] deps = module_depsolving_wrapper(session, deps, strict=strict) From edd92d65ae19d865bb1d024c79e8dbf5cf7352f2 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 17:54:18 +0200 Subject: [PATCH 063/106] handlers/modules: get rid of hardcoded dependency, fix typo --- rida/scheduler/handlers/modules.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 91a4e078..e506234f 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -27,6 +27,7 @@ import rida.builder import rida.database import rida.pdc import logging +import os log = logging.getLogger(__name__) @@ -68,12 +69,14 @@ def wait(config, session, msg): dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) build.buildroot_task_id = builder.buildroot_prep() - buildroot_add_dependency(["f24-build",]) # XXX: hack remove once we have dependencies + log.debug("Adding dependencies %s into buildroot for module %s" % (dependencies, module_info)) builder.buildroot_add_dependency(dependencies) srpm = builder.get_disttag_srpm(disttag="%s" % get_rpm_release_from_tag(tag)) - builder.build(srpm) + task_id = builder.build(srpm) + builder.wait_task(task_id) + artifact = get_artifact_from_srpm(srpm) - bulder.add_artifact(artifact) + builder.buildroot_add_artifacts([artifact,]) builder.buildroot_ready(artifacts=[artifact,]) build.transition(config, state="build") # Wait for the buildroot to be ready. session.commit() From 879eca579aeb391d4df5dc48033be32716a1148f Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 17:54:42 +0200 Subject: [PATCH 064/106] pass over correct scmurl, and valid artifact name --- rida/scheduler/handlers/repos.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 76b7c8bb..cb550728 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -52,12 +52,12 @@ def done(config, session, msg): builder.buildroot_resume() for component_build in unbuilt_components: - scmurl = "{dist_git}/rpms/{package}?#{gitref}".format( + scmurl = "{dist_git}/{package}?#{gitref}".format( dist_git=config.rpms_default_repository, package=component_build.package, gitref=component_build.gitref, # This is the update stream ) - artifact_name = 'TODO' component_build.state = koji.BUILD_STATES['BUILDING'] - component_build.task_id = builder.build(artifact_name, scmurl) + log.debug("Using scmurl=%s for package=%s" % (scmurl, component_build.package)) + component_build.task_id = builder.build(artifact=component_build.package, source=scmurl) session.commit() From bae1ecc150f0b959f71a3bbcc78e8dc5bdcc6edf Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 17:55:11 +0200 Subject: [PATCH 065/106] update test-pdc to work with new server --- test-pdc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test-pdc.py b/test-pdc.py index c5156377..b4f66bc2 100755 --- a/test-pdc.py +++ b/test-pdc.py @@ -6,17 +6,17 @@ from rida.config import Config cfg = Config() -cfg.pdc_url = "http://localhost:8000/rest_api/v1" +cfg.pdc_url = "http://modularity.fedorainfracloud.org:8080/rest_api/v1" cfg.pdc_insecure = True cfg.pdc_develop = True pdc_session = get_pdc_client_session(cfg) -module = get_module(pdc_session, {'name': 'testmodule', 'version': '4.3.42', 'release': '0'}) +module = get_module(pdc_session, {'name': 'testmodule', 'version': '4.3.43', 'release': '1'}) if module: print ("pdc_data=%s" % str(module)) - print ("deps=[%s]" % ", ".join(get_module_dependencies(pdc_session, module))) - print ("build_deps=[%s]" % ", ".join(get_module_build_dependencies(pdc_session, module))) + print ("deps=%s" % get_module_runtime_dependencies(pdc_session, module)) + print ("build_deps=%s" % get_module_build_dependencies(pdc_session, module)) print ("tag=%s" % get_module_tag(pdc_session, module)) else: print ('module was not found') From 0fbe160af038c5194b7d78fdc9c9571301fe5a30 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 19:42:30 +0200 Subject: [PATCH 066/106] waiting for right task_id, cleanup of code --- rida.conf | 6 +++--- rida/scheduler/handlers/modules.py | 11 ++++++++--- submit-build.json | 2 +- tests/test_scheduler/test_repo_done.py | 1 + 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/rida.conf b/rida.conf index d1ea9a31..dd332951 100644 --- a/rida.conf +++ b/rida.conf @@ -1,12 +1,12 @@ [DEFAULT] system = koji messaging = fedmsg -koji_config = ~/.koji/config +koji_config = /etc/rida/koji.conf # See https://fedoraproject.org/wiki/Koji/WritingKojiCode#Profiles -koji_profile = staging +koji_profile = koji koji_arches = ["x86_64"] db = sqlite:///rida.db -pdc_url = http://modularity.fedorainfracloud.org:8080/rest_api/v1 +pdc_url = http://modularity.fedorainfracloud.org:8080/rest_api/v1/ pdc_insecure = True pdc_develop = True scmurls = ["git://pkgs.stg.fedoraproject.org/modules/"] diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index e506234f..a8e8a25b 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -29,6 +29,7 @@ import rida.pdc import logging import os +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -64,19 +65,23 @@ def wait(config, session, msg): # associated with which koji tag, so that when their repos are regenerated # in koji we can figure out which for which module build that event is # relevant. + log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag - dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info) + + dependencies = rida.pdc.get_module_build_dependencies(pdc_session, module_info) builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) build.buildroot_task_id = builder.buildroot_prep() log.debug("Adding dependencies %s into buildroot for module %s" % (dependencies, module_info)) builder.buildroot_add_dependency(dependencies) + # inject dist-tag into buildroot srpm = builder.get_disttag_srpm(disttag="%s" % get_rpm_release_from_tag(tag)) - task_id = builder.build(srpm) + task_id = builder.build(artifact_name="module-build-macros", source=srpm) builder.wait_task(task_id) artifact = get_artifact_from_srpm(srpm) - builder.buildroot_add_artifacts([artifact,]) + builder.buildroot_add_artifacts([artifact,]) # pretty much srpm filename builder.buildroot_ready(artifacts=[artifact,]) + build.transition(config, state="build") # Wait for the buildroot to be ready. session.commit() diff --git a/submit-build.json b/submit-build.json index 7873ee98..b31328e9 100644 --- a/submit-build.json +++ b/submit-build.json @@ -1,3 +1,3 @@ { - "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#020ea37251df5019fde9e7899d2f7d7a987dfbf5" + "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#5188d22b255c9f54798926959f43967a057ca690" } diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py index 09e8db65..8cdd4004 100644 --- a/tests/test_scheduler/test_repo_done.py +++ b/tests/test_scheduler/test_repo_done.py @@ -33,6 +33,7 @@ class TestRepoDone(unittest.TestCase): self.config.rpms_default_repository = 'dist_git_url' self.config.koji_profile = 'staging' # TODO - point at a fake test config + self.session = mock.Mock() self.fn = rida.scheduler.handlers.repos.done From 45c869a37ba28df29a69b8eb0520ba01d4d1f6f5 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 19:42:59 +0200 Subject: [PATCH 067/106] builder: implement wait_task --- rida/builder.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index ccaa389a..5c63178a 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -42,6 +42,7 @@ import time import random import string +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) # TODO: read defaults from rida's config @@ -363,8 +364,19 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules :param task_id :return - task result object """ - log.info("Waiting for task_id=%s" % task_id) - return self.koji_session.getTaskResult(task_id) + start = time.time() + timeout = 60 # minutes + + while True: + if (time.time() - start) >= (timeout * 60.0): + break + try: + log.debug("Waiting for task_id=%s to finish" % task_id) + return self.koji_session.getTaskResult(task_id) + + except koji.GenericError: + time.sleep(30) + return 1 def build(self, artifact_name, source): """ From 3e36e339056e13907941bb689b5c5b9a947ed699 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 19:43:19 +0200 Subject: [PATCH 068/106] pdc: is_module_str needs to check for unicode as well --- rida/pdc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/pdc.py b/rida/pdc.py index 04539335..40e4085a 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -68,7 +68,7 @@ def get_variant_dict(data): return isinstance(data, modulemd.ModuleMetadata) def is_module_str(data): - return isinstance(data, str) + return isinstance(data, str) or isinstance(data, unicode) result = None From 787f06d7efb06b9d5e8a9cf68b7dcd02a8364a94 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Wed, 20 Jul 2016 19:44:46 +0200 Subject: [PATCH 069/106] add logging.BasicConfig --- rida/scheduler/handlers/components.py | 1 + rida/scheduler/handlers/repos.py | 1 + 2 files changed, 2 insertions(+) diff --git a/rida/scheduler/handlers/components.py b/rida/scheduler/handlers/components.py index fcd1c5f6..68875020 100644 --- a/rida/scheduler/handlers/components.py +++ b/rida/scheduler/handlers/components.py @@ -31,6 +31,7 @@ import rida.pdc import koji +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) def _finalize(config, session, msg, state): diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index cb550728..2f1ad620 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -29,6 +29,7 @@ import rida.pdc import logging import koji +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) From e218ac4fa84e605430387cf7b452c153bd810018 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 14:35:24 -0400 Subject: [PATCH 070/106] Enhance this repr. --- rida/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/database.py b/rida/database.py index 79f2cf4f..4fe2d71e 100644 --- a/rida/database.py +++ b/rida/database.py @@ -256,5 +256,5 @@ class ComponentBuild(Base): } def __repr__(self): - return "" % ( - self.package, self.module_id, self.state) + return "" % ( + self.package, self.module_id, self.state, self.task_id) From dcc8a8b732b2fbdf1f385f402fe888827ca52dac Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 14:35:37 -0400 Subject: [PATCH 071/106] Another log statement to try and figure out whats going on. --- rida/scheduler/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index fd8a8924..0ffd80a6 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -227,6 +227,7 @@ class Poller(threading.Thread): koji.TASK_STATES['CANCELED'], koji.TASK_STATES['FAILED'], ) + log.info(" task %r is in state %r" % (component_build.task_id, task_info['state'])) if task_info['state'] in dead_states: # Fake a fedmsg message on our internal queue self.outgoing_work_queue.put({ From 544b99ba093d6b8b8d92bfedc678ee077aa75623 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 14:35:45 -0400 Subject: [PATCH 072/106] A TODO for later.. --- rida/scheduler/handlers/modules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index a8e8a25b..f20d3cc0 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -77,6 +77,9 @@ def wait(config, session, msg): # inject dist-tag into buildroot srpm = builder.get_disttag_srpm(disttag="%s" % get_rpm_release_from_tag(tag)) task_id = builder.build(artifact_name="module-build-macros", source=srpm) + + # TODO -- this has to go eventually.. otherwise, we can only build one + # module at a time and that just won't scale. builder.wait_task(task_id) artifact = get_artifact_from_srpm(srpm) From d08c7b936b8b580c6150dfd49820e10f347b14e9 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 14:42:56 -0400 Subject: [PATCH 073/106] Keep and use the full scm url instead of reconstructing it. --- rida.py | 5 +++-- rida/database.py | 2 +- rida/scheduler/handlers/repos.py | 15 ++++++++------- tests/test_scheduler/test_repo_done.py | 4 ++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/rida.py b/rida.py index eac87544..f62373fb 100755 --- a/rida.py +++ b/rida.py @@ -133,13 +133,14 @@ def submit_build(): pkg["commit"] = rida.scm.SCM(pkg["repository"]).get_latest() except Exception as e: return failure("Failed to get the latest commit: %s" % pkgname, 422) - if not rida.scm.SCM(pkg["repository"] + "?#" + pkg["commit"]).is_available(): + full_url = pkg["repository"] + "?#" + pkg["commit"] + if not rida.scm.SCM(full_url).is_available(): return failure("Cannot checkout %s" % pkgname, 422) build = rida.database.ComponentBuild( module_id=module.id, package=pkgname, format="rpms", - gitref=pkg["commit"], # TODO - re-evaluate this w.r.t. supported branches + scmurl=full_url, ) db.session.add(build) module.modulemd = mmd.dumps() diff --git a/rida/database.py b/rida/database.py index 4fe2d71e..d87ddb4f 100644 --- a/rida/database.py +++ b/rida/database.py @@ -229,7 +229,7 @@ class ComponentBuild(Base): __tablename__ = "component_builds" id = Column(Integer, primary_key=True) package = Column(String, nullable=False) - gitref = Column(String, nullable=False) + scmurl = Column(String, nullable=False) # XXX: Consider making this a proper ENUM format = Column(String, nullable=False) task_id = Column(Integer) # This is the id of the build in koji diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 2f1ad620..43341c5f 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -53,12 +53,13 @@ def done(config, session, msg): builder.buildroot_resume() for component_build in unbuilt_components: - scmurl = "{dist_git}/{package}?#{gitref}".format( - dist_git=config.rpms_default_repository, - package=component_build.package, - gitref=component_build.gitref, # This is the update stream - ) component_build.state = koji.BUILD_STATES['BUILDING'] - log.debug("Using scmurl=%s for package=%s" % (scmurl, component_build.package)) - component_build.task_id = builder.build(artifact=component_build.package, source=scmurl) + log.debug("Using scmurl=%s for package=%s" % ( + component_build.scmurl, + component_build.package, + )) + component_build.task_id = builder.build( + artifact=component_build.package, + source=component_build.scmurl, + ) session.commit() diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py index 8cdd4004..85afea6d 100644 --- a/tests/test_scheduler/test_repo_done.py +++ b/tests/test_scheduler/test_repo_done.py @@ -58,7 +58,7 @@ class TestRepoDone(unittest.TestCase): """ component_build = mock.Mock() component_build.package = 'foo' - component_build.gitref = 'beef' + component_build.scmurl = 'full_scm_url' component_build.state = None module_build = mock.Mock() module_build.component_builds = [component_build] @@ -69,4 +69,4 @@ class TestRepoDone(unittest.TestCase): 'msg': {'tag': 'no matches for this...'}, } self.fn(config=self.config, session=self.session, msg=msg) - build_fn.assert_called_once_with('TODO', 'dist_git_url/rpms/foo?#beef') + build_fn.assert_called_once_with('TODO', 'full_scm_url') From caaeb4c7bc558ab7e60507798a829feb22f0e83a Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 14:48:08 -0400 Subject: [PATCH 074/106] Use munch instead of _get_opts_from_dict. --- requirements.txt | 1 + rida/builder.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index f3b58350..2c9b107a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ modulemd pyOpenSSL kobo koji +munch diff --git a/rida/builder.py b/rida/builder.py index 5c63178a..279c3cc7 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -42,6 +42,8 @@ import time import random import string +import munch + logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -153,13 +155,6 @@ class Builder: else: raise ValueError("Builder backend='%s' not recognized" % backend) -def _get_opts_from_dict(data): - """koji requires config in optparse opts style""" - config = OptionParser() - opts, _ = config.parse_args() - for key, value in data.iteritems(): - setattr(opts, key, value) - return opts class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ @@ -278,8 +273,14 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules @staticmethod def get_session_from_config(config): - koji_config = _get_opts_from_dict(koji.read_config(profile_name=config.koji_profile, user_config=config.koji_config)) - koji_module = koji.get_profile_module(config.koji_profile, config=koji_config) + koji_config = munch.Munch(koji.read_config( + profile_name=config.koji_profile, + user_config=config.koji_config, + )) + koji_module = koji.get_profile_module( + config.koji_profile, + config=koji_config, + ) krbservice = getattr(koji_config, "krbservice", None) if krbservice: From fb968077a905228eeb22a76c619e402c85a67090 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 14:48:32 -0400 Subject: [PATCH 075/106] Make note of the **extra arguments. --- rida/builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rida/builder.py b/rida/builder.py index 279c3cc7..5d670d5a 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -148,6 +148,9 @@ class Builder: :param module : a module string e.g. 'testmodule-1.0' :param backend: a string representing backend e.g. 'koji' :param config: instance of rida.config.Config + + Any additional arguments are optional extras which can be passed along + and are implementation-dependent. """ if backend == "koji": From 8b3ed86fa823a4eb4d7171de7e7c7b4867792fc8 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 15:50:48 -0400 Subject: [PATCH 076/106] Change the names of these classmethods to be a little more generic. --- rida/database.py | 27 ++++++++++++------------ rida/scheduler/handlers/components.py | 2 +- rida/scheduler/handlers/modules.py | 2 +- rida/scheduler/handlers/repos.py | 3 +-- tests/test_scheduler/test_module_wait.py | 6 +++--- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/rida/database.py b/rida/database.py index d87ddb4f..f0998656 100644 --- a/rida/database.py +++ b/rida/database.py @@ -146,10 +146,10 @@ class ModuleBuild(Base): raise ValueError("%s: %s, not in %r" % (key, field, BUILD_STATES)) @classmethod - def from_fedmsg(cls, session, msg): - if '.module.' not in msg['topic']: - raise ValueError("%r is not a module message." % msg['topic']) - return session.query(cls).filter(cls.id==msg['msg']['id']).first() + def from_module_event(cls, session, event): + if '.module.' not in event['topic']: + raise ValueError("%r is not a module message." % event['topic']) + return session.query(cls).filter(cls.id==event['msg']['id']).first() @classmethod def create(cls, session, conf, name, version, release, modulemd): @@ -188,19 +188,20 @@ class ModuleBuild(Base): .filter_by(state=BUILD_STATES[state]).all() @classmethod - def get_active_by_koji_tag(cls, session, koji_tag): + def from_repo_done_event(cls, session, event): """ Find the ModuleBuilds in our database that should be in-flight... ... for a given koji tag. There should be at most one. """ - query = session.query(rida.database.ModuleBuild)\ - .filter_by(koji_tag=koji_tag)\ - .filter_by(state=BUILD_STATES["build"]) + tag = event['msg']['tag'].strip('-build') + query = session.query(cls)\ + .filter(cls.koji_tag==tag)\ + .filter(cls.state==BUILD_STATES["build"]) count = query.count() if count > 1: - raise RuntimeError("%r module builds in flight for %r" % (count, koji_tag)) + raise RuntimeError("%r module builds in flight for %r" % (count, tag)) return query.first() @@ -240,10 +241,10 @@ class ComponentBuild(Base): module_build = relationship('ModuleBuild', backref='component_builds', lazy=False) @classmethod - def from_fedmsg(cls, session, msg): - if 'component.state.change' not in msg['topic'] and '.buildsys.build.state.change' not in msg['topic']: - raise ValueError("%r is not a koji message." % msg['topic']) - return session.query(cls).filter(cls.task_id==msg['msg']['task_id']).first() + def from_component_event(cls, session, event): + if 'component.state.change' not in event['topic'] and '.buildsys.build.state.change' not in event['topic']: + raise ValueError("%r is not a koji message." % event['topic']) + return session.query(cls).filter(cls.task_id==event['msg']['task_id']).first() def json(self): return { diff --git a/rida/scheduler/handlers/components.py b/rida/scheduler/handlers/components.py index 68875020..d81b5224 100644 --- a/rida/scheduler/handlers/components.py +++ b/rida/scheduler/handlers/components.py @@ -38,7 +38,7 @@ def _finalize(config, session, msg, state): """ Called whenever a koji build completes or fails. """ # First, find our ModuleBuild associated with this repo, if any. - component_build = rida.database.ComponentBuild.from_fedmsg(session, msg) + component_build = rida.database.ComponentBuild.from_component_event(session, msg) if not component_build: template = "We have no record of {name}-{version}-{release}" log.debug(template.format(**msg['msg'])) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index f20d3cc0..109dc778 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -48,7 +48,7 @@ def wait(config, session, msg): The kicking off of individual component builds is handled elsewhere, in rida.schedulers.handlers.repos. """ - build = rida.database.ModuleBuild.from_fedmsg(session, msg) + build = rida.database.ModuleBuild.from_module_event(session, msg) module_info = build.json() if module_info['state'] != rida.BUILD_STATES["wait"]: # XXX: not sure why did we get here from state == 2 (building) FIXTHIS diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 43341c5f..0c0583f8 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -38,8 +38,7 @@ def done(config, session, msg): # First, find our ModuleBuild associated with this repo, if any. tag = msg['msg']['tag'].strip('-build') - module_build = rida.database.ModuleBuild.get_active_by_koji_tag( - session, koji_tag=tag) + module_build = rida.database.ModuleBuild.from_repo_done_event(session, msg) if not module_build: log.info("No module build found associated with koji tag %r" % tag) return diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index 5f60c3f6..be7e4db5 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -34,9 +34,9 @@ class TestModuleWait(unittest.TestCase): self.fn = rida.scheduler.handlers.modules.wait @mock.patch('rida.builder.KojiModuleBuilder') - @mock.patch('rida.database.ModuleBuild.from_fedmsg') + @mock.patch('rida.database.ModuleBuild.from_module_event') @mock.patch('rida.pdc.get_pdc_client_session') - def test_wait_basic(self, pdc, from_fedmsg, KojiModuleBuilder): + def test_wait_basic(self, pdc, from_module_event, KojiModuleBuilder): builder = mock.Mock() KojiModuleBuilder.return_value = builder mocked_module_build = mock.Mock() @@ -45,7 +45,7 @@ class TestModuleWait(unittest.TestCase): 'version': 1, 'release': 1, } - from_fedmsg.return_value = mocked_module_build + from_module_event.return_value = mocked_module_build msg = { 'topic': 'org.fedoraproject.prod.rida.module.state.change', From 69a67abc3e45f7ab7859222bca8b5d102522c722 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 15:52:36 -0400 Subject: [PATCH 077/106] Update apidoc. --- rida/pdc.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rida/pdc.py b/rida/pdc.py index 40e4085a..f15d0331 100644 --- a/rida/pdc.py +++ b/rida/pdc.py @@ -121,6 +121,9 @@ def get_module(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info: pdc variant_dict, str, mmd or module dict + :param strict: Normally this function returns None if no module can be + found. If strict=True, then a ValueError is raised. + :return final list of module_info which pass repoclosure """ @@ -144,6 +147,8 @@ def get_module_tag(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info: list of module_info dicts + :param strict: Normally this function returns None if no module can be + found. If strict=True, then a ValueError is raised. :return: koji tag string """ # TODO -- get this from PDC some day... for now, we're just going to @@ -184,7 +189,8 @@ def get_module_runtime_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance :param module_infos : a dict containing filters for pdc - :param strict=False : don't raise exception if None is returned + :param strict: Normally this function returns None if no module can be + found. If strict=True, then a ValueError is raised. Example minimal module_info {'variant_name': module_name, 'variant_version': module_version, 'variant_type': 'module'} """ @@ -202,6 +208,8 @@ def get_module_build_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info : a dict containing filters for pdc + :param strict: Normally this function returns None if no module can be + found. If strict=True, then a ValueError is raised. :return final list of module_infos which pass repoclosure Example minimal module_info {'variant_name': module_name, 'variant_version': module_version, 'variant_type': 'module'} From e81c6d79274d3d8d3fdbe43a08e534d2521f5377 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 16:01:22 -0400 Subject: [PATCH 078/106] Remove unused import. --- rida/builder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rida/builder.py b/rida/builder.py index 5d670d5a..b63f48bc 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -32,7 +32,6 @@ from abc import ABCMeta, abstractmethod import logging import os -from optparse import OptionParser from kobo.shortcuts import run import koji import tempfile From e256abe60e53dcfdb11ab89b222133e42df703ba Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 16:01:35 -0400 Subject: [PATCH 079/106] Straighten out this name/object confusion. --- rida/builder.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index b63f48bc..314fe1f7 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -170,6 +170,7 @@ class KojiModuleBuilder(GenericBuilder): :param tag_name: name of tag for given module """ self.module_str = module + self.tag_name = tag_name self.__prep = False log.debug("Using koji profile %r" % config.koji_profile) log.debug ("Using koji_config: %s" % config.koji_config) @@ -179,13 +180,14 @@ class KojiModuleBuilder(GenericBuilder): if not self.arches: raise ValueError("No koji_arches specified in the config.") - self.module_tag = tag_name - self.module_build_tag = "%s-build" % tag_name - self.module_target = tag_name + # These eventually get populated when buildroot_{prep,resume} is called + self.module_tag = None + self.module_build_tag = None + self.module_target = None def __repr__(self): return "" % ( - self.module_str, self.module_tag) + self.module_str, self.tag_name) def buildroot_ready(self, artifacts=None): target_info = self.koji_session.getBuildTarget(self.module_target) @@ -319,15 +321,15 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules """ Resume existing buildroot. Sets __prep=True """ - chktag = self.koji_session.getTag(self.module_tag) + chktag = self.koji_session.getTag(self.tag_name) if not chktag: - raise SystemError("Tag %s doesn't exist" % self.module_tag) - chkbuildtag = self.koji_session.getTag(self.module_build_tag) + raise SystemError("Tag %s doesn't exist" % self.tag_name) + chkbuildtag = self.koji_session.getTag(self.tag_name + "-build") if not chkbuildtag: - raise SystemError("Build Tag %s doesn't exist" % self.module_build_tag) - chktarget = self.koji_session.getBuildTarget(self.module_target) + raise SystemError("Build Tag %s doesn't exist" % self.tag_name + "-build") + chktarget = self.koji_session.getBuildTarget(self.tag_name) if not chktarget: - raise SystemError("Target %s doesn't exist" % self.module_target) + raise SystemError("Target %s doesn't exist" % self.tag_name) self.module_tag = chktag self.module_build_tag = chkbuildtag self.module_target = chktarget @@ -339,14 +341,16 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules :param module_deps_tags: a tag names of our build requires :param module_deps_tags: a tag names of our build requires """ - self.module_tag = self._koji_create_tag(self.module_tag, perm="admin") # returns tag obj - self.module_build_tag = self._koji_create_tag(self.module_build_tag, self.arches, perm="admin") + self.module_tag = self._koji_create_tag( + self.tag_name, perm="admin") # returns tag obj + self.module_build_tag = self._koji_create_tag( + self.tag_name + "-build", self.arches, perm="admin") groups = KOJI_DEFAULT_GROUPS # TODO: read from config if groups: self._koji_add_groups_to_tag(self.module_build_tag, groups) - self.module_target = self._koji_add_target(self.module_target, self.module_build_tag, self.module_tag) + self.module_target = self._koji_add_target(self.tag_name, self.module_build_tag, self.module_tag) self.__prep = True log.info("%r buildroot prepared." % self) From 618bbbe34f27d1ab188ef670578988d36e107c8c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 20 Jul 2016 16:06:36 -0400 Subject: [PATCH 080/106] Log tidying. --- rida.conf | 4 ++-- rida/builder.py | 2 +- rida/scheduler/main.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rida.conf b/rida.conf index dd332951..c17aa37f 100644 --- a/rida.conf +++ b/rida.conf @@ -1,9 +1,9 @@ [DEFAULT] system = koji messaging = fedmsg -koji_config = /etc/rida/koji.conf +koji_config = ~/.koji/config # See https://fedoraproject.org/wiki/Koji/WritingKojiCode#Profiles -koji_profile = koji +koji_profile = staging koji_arches = ["x86_64"] db = sqlite:///rida.db pdc_url = http://modularity.fedorainfracloud.org:8080/rest_api/v1/ diff --git a/rida/builder.py b/rida/builder.py index 314fe1f7..03977769 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -291,7 +291,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules koji_config.krbservice = krbservice address = koji_config.server - log.info("Connecting to koji %r, %r" % (address, koji_config)) + log.info("Connecting to koji %r" % address) koji_session = koji.ClientSession(address, opts=vars(koji_config)) authtype = koji_config.authtype diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 0ffd80a6..8b28f7c6 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -175,9 +175,10 @@ class MessageWorker(threading.Thread): # Execute our chosen handler with rida.database.Database(config) as session: - log.info(" %r: %s, %s" % (handler, msg['topic'], msg['msg_id'])) + log.info(" %s: %s, %s" % (handler.__name__, msg['topic'], msg['msg_id'])) handler(config, session, msg) + class Poller(threading.Thread): def __init__(self, outgoing_work_queue, *args, **kwargs): self.outgoing_work_queue = outgoing_work_queue From a52037b67a92e41899db7ee4c97d06310e044c92 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 21 Jul 2016 09:23:07 -0400 Subject: [PATCH 081/106] Log around waiting.. --- rida/builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rida/builder.py b/rida/builder.py index 03977769..8a78fa51 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -374,6 +374,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules start = time.time() timeout = 60 # minutes + log.info("Waiting for task_id=%s to finish" % task_id) while True: if (time.time() - start) >= (timeout * 60.0): break @@ -383,6 +384,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules except koji.GenericError: time.sleep(30) + log.info("Done waiting for task_id=%s to finish" % task_id) return 1 def build(self, artifact_name, source): From b959d044eb2486a63dd618655fe14fa0b623e886 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 21 Jul 2016 09:23:21 -0400 Subject: [PATCH 082/106] Make this a little more readable. --- rida/builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 8a78fa51..0b310437 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -416,8 +416,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules source = "%s/%s" % (serverdir, os.path.basename(source)) task_id = self.koji_session.build(source, self.module_target['name']) - log.info("%r submitted build of %s (task_id=%s)" % ( - self, source, task_id)) + log.info("submitted build of %s (task_id=%s), via %s" % ( + source, task_id, self)) return task_id def _get_tag(self, tag, strict=True): From 90abfd8901cba77770716d2fa8e278041da5b6a4 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 21 Jul 2016 09:23:35 -0400 Subject: [PATCH 083/106] Include state_name in the json representation of models. --- rida/database.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index f0998656..c24fd0ca 100644 --- a/rida/database.py +++ b/rida/database.py @@ -212,6 +212,7 @@ class ModuleBuild(Base): 'version': self.version, 'release': self.release, 'state': self.state, + 'state_name': INVERSE_BUILD_STATES[self.state], # This is too spammy.. #'modulemd': self.modulemd, @@ -247,7 +248,7 @@ class ComponentBuild(Base): return session.query(cls).filter(cls.task_id==event['msg']['task_id']).first() def json(self): - return { + retval = { 'id': self.id, 'package': self.package, 'format': self.format, @@ -256,6 +257,18 @@ class ComponentBuild(Base): 'module_build': self.module_id, } + try: + # Koji is py2 only, so this fails if the main web process is + # running on py3. + import koji + retval['state_name'] = koji.BUILD_STATES[self.state] + except ImportError: + pass + + return retval + + + def __repr__(self): return "" % ( self.package, self.module_id, self.state, self.task_id) From 747f5a3514bb860f94b51dc6df2e55eddd0ea79b Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 21 Jul 2016 09:25:53 -0400 Subject: [PATCH 084/106] Revert unintentional changes from 618bbbe34f27d1ab188ef670578988d36e107c8c --- rida.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida.conf b/rida.conf index c17aa37f..dd332951 100644 --- a/rida.conf +++ b/rida.conf @@ -1,9 +1,9 @@ [DEFAULT] system = koji messaging = fedmsg -koji_config = ~/.koji/config +koji_config = /etc/rida/koji.conf # See https://fedoraproject.org/wiki/Koji/WritingKojiCode#Profiles -koji_profile = staging +koji_profile = koji koji_arches = ["x86_64"] db = sqlite:///rida.db pdc_url = http://modularity.fedorainfracloud.org:8080/rest_api/v1/ From 5b06b64833af8109d6058de2fd82371771e97d45 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 21 Jul 2016 10:31:36 -0400 Subject: [PATCH 085/106] Careful when self.state is None. --- rida/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/database.py b/rida/database.py index c24fd0ca..88a8ce48 100644 --- a/rida/database.py +++ b/rida/database.py @@ -261,7 +261,7 @@ class ComponentBuild(Base): # Koji is py2 only, so this fails if the main web process is # running on py3. import koji - retval['state_name'] = koji.BUILD_STATES[self.state] + retval['state_name'] = koji.BUILD_STATES.get(self.state) except ImportError: pass From 9d1e03ed9c9c2ff8ba03f4208ada2fe40e001b23 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 14:02:51 +0200 Subject: [PATCH 086/106] koji: fixed add_artifact - pass tag_id instead of dict --- rida/builder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 0b310437..d358611b 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -363,8 +363,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules # TODO: import /usr/bin/koji's TaskWatcher() log.info("%r adding artifacts %r" % (self, artifacts)) for nvr in artifacts: - # we do need taginfo dict not the string _get_tag() - self.koji_session.tagBuild(self._get_tag(self.module_build_tag), nvr, force=True) + self.koji_session.tagBuild(self._get_tag(self.module_build_tag)['id'], nvr, force=True) def wait_task(self, task_id): """ From 3b636cd2d05366149ba606d788c4d9a459010f8d Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 14:03:11 +0200 Subject: [PATCH 087/106] add . at beginning of dist-tag --- rida/scheduler/handlers/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 109dc778..8e7a6aa3 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -75,7 +75,7 @@ def wait(config, session, msg): log.debug("Adding dependencies %s into buildroot for module %s" % (dependencies, module_info)) builder.buildroot_add_dependency(dependencies) # inject dist-tag into buildroot - srpm = builder.get_disttag_srpm(disttag="%s" % get_rpm_release_from_tag(tag)) + srpm = builder.get_disttag_srpm(disttag=".%s" % get_rpm_release_from_tag(tag)) task_id = builder.build(artifact_name="module-build-macros", source=srpm) # TODO -- this has to go eventually.. otherwise, we can only build one From 6c6a5b40b671f0aa29db75bb4bbb15191e30661c Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 14:36:53 +0200 Subject: [PATCH 088/106] builder/repos: use artifact_name when reffering to a name and artifact when you refer NameVersionRelase * fixed wait_buildroot - pass buildInfos instad of nvrs * add few debug messages to repos.py --- rida/builder.py | 22 ++++++++++++---------- rida/scheduler/handlers/repos.py | 3 ++- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index d358611b..c5515c71 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -135,6 +135,7 @@ class GenericBuilder: :param artifact_name : a crucial, since we can't guess a valid srpm name without having the exact buildroot (collections/macros) used e.g. for whitelisting packages + artifact_name is used to distinguish from artifact (e.g. package x nvr) :param source : a scmurl to repository with receipt (e.g. spec) """ raise NotImplementedError() @@ -181,27 +182,27 @@ class KojiModuleBuilder(GenericBuilder): raise ValueError("No koji_arches specified in the config.") # These eventually get populated when buildroot_{prep,resume} is called - self.module_tag = None - self.module_build_tag = None - self.module_target = None + self.module_tag = None # string + self.module_build_tag = None # string + self.module_target = None # A koji target dict def __repr__(self): return "" % ( self.module_str, self.tag_name) def buildroot_ready(self, artifacts=None): - target_info = self.koji_session.getBuildTarget(self.module_target) - assert target_info, "Invalid build target" + assert self.module_target, "Invalid build target" timeout = 120 # minutes see * 60 - tag_id = target_info['build_tag'] + tag_id = self.module_target['build_tag'] start = time.time() last_repo = None repo = self.koji_session.getRepo(tag_id) + builds = [ self.koji_session.getBuild(a) for a in artifacts or []] while True: - if artifacts and repo and repo != last_repo: - if koji.util.checkForBuilds(self.koji_session, tag_id, artifacts, repo['create_event'], latest=True): + if builds and repo and repo != last_repo: + if koji.util.checkForBuilds(self.koji_session, tag_id, builds, repo['create_event'], latest=True): return if (time.time() - start) >= (timeout * 60.0): @@ -211,7 +212,7 @@ class KojiModuleBuilder(GenericBuilder): last_repo = repo repo = self.koji_session.getRepo(tag_id) - if not artifacts: + if not builds: if repo != last_repo: return @@ -389,6 +390,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules def build(self, artifact_name, source): """ :param source : scmurl to spec repository + : param artifact_name: name of artifact (which we couldn't get from spec due involved macros) :return koji build task id """ # Taken from /usr/bin/koji @@ -472,7 +474,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules def _koji_create_tag(self, tag_name, arches=None, fail_if_exists=True, perm=None): - print ("Creating tag %s" % tag_name) + log.debug("Creating tag %s" % tag_name) chktag = self.koji_session.getTag(tag_name) if chktag and fail_if_exists: raise SystemError("Tag %s already exist" % tag_name) diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 0c0583f8..2ff80b3c 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -57,8 +57,9 @@ def done(config, session, msg): component_build.scmurl, component_build.package, )) + log.info("Building artifact_name=%s from source=%s" % (component_build.package. component_build.scmurl)) component_build.task_id = builder.build( - artifact=component_build.package, + artifact_name=component_build.package, source=component_build.scmurl, ) session.commit() From d507dcdb0d689d363649910d452bc4af90cdc87e Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 14:40:43 +0200 Subject: [PATCH 089/106] fixed typo dot -> comma in repos.py --- rida/scheduler/handlers/repos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scheduler/handlers/repos.py b/rida/scheduler/handlers/repos.py index 2ff80b3c..2c76b947 100644 --- a/rida/scheduler/handlers/repos.py +++ b/rida/scheduler/handlers/repos.py @@ -57,7 +57,7 @@ def done(config, session, msg): component_build.scmurl, component_build.package, )) - log.info("Building artifact_name=%s from source=%s" % (component_build.package. component_build.scmurl)) + log.info("Building artifact_name=%s from source=%s" % (component_build.package, component_build.scmurl)) component_build.task_id = builder.build( artifact_name=component_build.package, source=component_build.scmurl, From 1f521d9eda10cd6271c5ec7ccfce4d0081b07a03 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 15:22:34 +0200 Subject: [PATCH 090/106] builder: fix adding groups --- rida/builder.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index c5515c71..c2651923 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -454,21 +454,24 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules :param build_tag_name :param groups: A dict {'group' : [package, ...]} """ + log.debug("Adding groups=%s to tag=%s" % (groups.keys(), dest_tag)) if groups and not isinstance(groups, dict): raise ValueError("Expected dict {'group' : [str(package1), ...]") dest_tag = self._get_tag(dest_tag)['name'] - groups = dict([ + existing_groups = dict([ (p['name'], p['group_id']) for p in self.koji_session.getTagGroups(dest_tag, inherit=False) ]) + for group, packages in groups.iteritems(): - group_id = groups.get(group, None) + group_id = existing_groups.get(group, None) if group_id is not None: log.warning("Group %s already exists for tag %s" % (group, dest_tag)) continue self.koji_session.groupListAdd(dest_tag, group) + log.debug("Adding %d packages into group=%s tag=%s" % (len(packages), group, dest_tag)) for pkg in packages: self.koji_session.groupPackageListAdd(dest_tag, group, pkg) From 257c3055146e0e198490dad6b754558527b8a994 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 15:57:23 +0200 Subject: [PATCH 091/106] builder/koji: use retry for add_group_packages --- rida/builder.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rida/builder.py b/rida/builder.py index c2651923..adc34140 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -42,6 +42,7 @@ import random import string import munch +from OpenSSL.SSL import SysCallError logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -158,6 +159,15 @@ class Builder: else: raise ValueError("Builder backend='%s' not recognized" % backend) +def retry(callback, **kwargs): + attempt = 0 + while True: + try: + callback(**kwargs) + break + except SysCallError: + attempt += 1 + log.debug("Retry(): attempt=%d retrying callback." % attempt) class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ @@ -349,7 +359,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules groups = KOJI_DEFAULT_GROUPS # TODO: read from config if groups: - self._koji_add_groups_to_tag(self.module_build_tag, groups) + retry(self._koji_add_groups_to_tag, dest_tag=self.module_build_tag, groups=groups) self.module_target = self._koji_add_target(self.tag_name, self.module_build_tag, self.module_tag) self.__prep = True From b0c4fbb11ca59c4238a9b832fa424572a2be9629 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:17:04 +0200 Subject: [PATCH 092/106] builder.py - fix dist-tag srpm creation, add option to pre-install artifact in buildroot --- rida/builder.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index adc34140..8c8e20c2 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -40,6 +40,7 @@ import datetime import time import random import string +import kobo.rpmlib import munch from OpenSSL.SSL import SysCallError @@ -123,9 +124,10 @@ class GenericBuilder: raise NotImplementedError() @abstractmethod - def buildroot_add_artifacts(self, artifacts): + def buildroot_add_artifacts(self, artifacts, install=False): """ :param artifacts: list of artifacts to be available in buildroot + :param install=False: pre-install artifact in buildroot (otherwise "make it available for install") add artifacts into buildroot, can be used to override buildroot macros """ raise NotImplementedError() @@ -161,13 +163,14 @@ class Builder: def retry(callback, **kwargs): attempt = 0 + log.debug("retry() calling %r(kwargs=%r)" % (callback, kwargs)) while True: try: callback(**kwargs) break except SysCallError: attempt += 1 - log.debug("Retry(): attempt=%d retrying callback." % attempt) + log.warn("retry(attempt=%d) calling %r(kwargs=%r)" % (attempt, callback, kwargs)) class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ @@ -259,9 +262,9 @@ It should NEVER be installed on any system as it will really mess up %build %install -mkdir -p %buildroot/%_rpmconfigdir/macro.d 2>/dev/null |: -echo %%dist %dist > %buildroot/%_rpmconfigdir/macro.modules -chmod 644 %buildroot/%_rpmconfigdir/macro.modules +mkdir -p %buildroot/%_rpmconfigdir/macros.d 2>/dev/null |: +echo %%dist %dist > %buildroot/%_rpmconfigdir/macros.d/macro.modules +chmod 644 %buildroot/%_rpmconfigdir/macros.d/macro.modules %files @@ -370,11 +373,23 @@ chmod 644 %buildroot/%_rpmconfigdir/macro.modules log.info("%r adding deps for %r" % (self, tags)) self._koji_add_many_tag_inheritance(self.module_build_tag, tags) - def buildroot_add_artifacts(self, artifacts): + def buildroot_add_artifacts(self, artifacts, install=False): + """ + :param artifacts - list of artifacts to add to buildroot + :param install=False - force install artifact (if it's not dragged in as dependency) + """ # TODO: import /usr/bin/koji's TaskWatcher() log.info("%r adding artifacts %r" % (self, artifacts)) + dest_tag = self._get_tag(self.module_build_tag)['id'] for nvr in artifacts: - self.koji_session.tagBuild(self._get_tag(self.module_build_tag)['id'], nvr, force=True) + self.koji_session.tagBuild(dest_tag, nvr, force=True) + if install: + # we usually want just srpm-build + for group in ('srpm-build',): + pkg_info = kobo.rpmlib.parse_nvr(nvr) + log.info("%r adding %s to group %s" % (self, pkg_info['name'], group)) + self.koji_session.groupPackageListAdd(dest_tag, group, pkg_info['name']) + def wait_task(self, task_id): """ From 4ae2eb0bbc7ffb22aea85411876dfa4331eef945 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:18:03 +0200 Subject: [PATCH 093/106] set default dist-git and lookaside to stage --- rida.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida.conf b/rida.conf index dd332951..a50b61e9 100644 --- a/rida.conf +++ b/rida.conf @@ -19,9 +19,9 @@ port = 5000 # Set to zero to disable polling polling_interval = 60 -rpms_default_repository = git://pkgs.fedoraproject.org/rpms/ +rpms_default_repository = git://pkgs.stg.fedoraproject.org/rpms/ rpms_allow_repository = False -rpms_default_cache = http://pkgs.fedoraproject.org/repo/pkgs/ +rpms_default_cache = http://pkgs.stg.fedoraproject.org/repo/pkgs/ rpms_allow_cache = False ssl_enabled = True From 5ef69b02fb5a8f8aa5cb1e414d1af8e9c78710d1 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:18:21 +0200 Subject: [PATCH 094/106] pre-install macros in buildroot --- rida/scheduler/handlers/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 8e7a6aa3..246aead5 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -52,7 +52,6 @@ def wait(config, session, msg): module_info = build.json() if module_info['state'] != rida.BUILD_STATES["wait"]: # XXX: not sure why did we get here from state == 2 (building) FIXTHIS - print("Invalid state %s for wait()" % module_info['state']) log.error("Invalid state %s for wait(). Msg=%s" % (module_info['state'], msg)) return log.info("Found module_info=%s from message" % module_info) @@ -81,9 +80,10 @@ def wait(config, session, msg): # TODO -- this has to go eventually.. otherwise, we can only build one # module at a time and that just won't scale. builder.wait_task(task_id) + # TODO -- do cleanup if this fails artifact = get_artifact_from_srpm(srpm) - builder.buildroot_add_artifacts([artifact,]) # pretty much srpm filename + builder.buildroot_add_artifacts([artifact,], install=True) # tag && add to srpm-build group builder.buildroot_ready(artifacts=[artifact,]) build.transition(config, state="build") # Wait for the buildroot to be ready. From fd814ef0432c3b17c8df96177a4ac0a5ec6b9a87 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:18:45 +0200 Subject: [PATCH 095/106] add msg_id when we fake message --- rida/scheduler/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 8b28f7c6..9eb44e56 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -234,6 +234,7 @@ class Poller(threading.Thread): self.outgoing_work_queue.put({ 'topic': 'org.fedoraproject.prod.buildsys.build.state.change', 'msg': { + 'msg_id': 'a faked internal message', 'task_id': component_build.task_id, 'new': koji.BUILD_STATES['FAILED'], }, From 1bd9b28b92534b06431eefb04244d529b1f990b1 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 22 Jul 2016 19:19:17 +0200 Subject: [PATCH 096/106] reflect bump of testmodule --- submit-build.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submit-build.json b/submit-build.json index b31328e9..8233168b 100644 --- a/submit-build.json +++ b/submit-build.json @@ -1,3 +1,3 @@ { - "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#5188d22b255c9f54798926959f43967a057ca690" + "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#d0c328b61a1b842bb5c400fc8eeb588af7746d2d" } From 5a75e8b7dfa683573f9bc325fb5fa534f0b06372 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 20:36:06 -0400 Subject: [PATCH 097/106] msg_id is required. --- rida/scheduler/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 8b28f7c6..ca7d9a5d 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -232,6 +232,7 @@ class Poller(threading.Thread): if task_info['state'] in dead_states: # Fake a fedmsg message on our internal queue self.outgoing_work_queue.put({ + 'msg_id': 'a faked internal message', 'topic': 'org.fedoraproject.prod.buildsys.build.state.change', 'msg': { 'task_id': component_build.task_id, From 08b694cd6b776f23d70c72eb9b87be56711e9d98 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 20:42:39 -0400 Subject: [PATCH 098/106] Remove erroneous block. --- rida/scheduler/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 651b55db..67a706a5 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -165,8 +165,6 @@ class MessageWorker(threading.Thread): handler = self.on_repo_change elif '.buildsys.build.state.change' in msg['topic']: handler = self.on_build_change[msg['msg']['new']] - elif 'rida.component.state.change' in msg['topic']: - handler = self.on_build_change[msg['msg']['new']] elif '.rida.module.state.change' in msg['topic']: handler = self.on_module_change[module_build_state_from_msg(msg)] else: From 9177f4ed5c9646771030c1471051fa9872c454e5 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 21:06:34 -0400 Subject: [PATCH 099/106] No need to worry about this condition. --- rida/scheduler/handlers/modules.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 246aead5..7a82f2a4 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -49,12 +49,15 @@ def wait(config, session, msg): in rida.schedulers.handlers.repos. """ build = rida.database.ModuleBuild.from_module_event(session, msg) + log.info("Found build=%r from message" % build) + module_info = build.json() - if module_info['state'] != rida.BUILD_STATES["wait"]: - # XXX: not sure why did we get here from state == 2 (building) FIXTHIS - log.error("Invalid state %s for wait(). Msg=%s" % (module_info['state'], msg)) - return - log.info("Found module_info=%s from message" % module_info) + if module_info['state'] != msg['msg']['state']: + log.warn("Note that retrieved module state %r " + "doesn't match message module state %r" % ( + module_info['state'], msg['msg']['state'])) + # This is ok.. it's a race condition we can ignore. + pass pdc_session = rida.pdc.get_pdc_client_session(config) tag = rida.pdc.get_module_tag(pdc_session, module_info, strict=True) From cb671a19da9aa2f63b3184eb03965cd12c76775c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 21:09:43 -0400 Subject: [PATCH 100/106] Fix dist-tag spec hack missed in b0c4fbb1. --- rida/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/builder.py b/rida/builder.py index 8c8e20c2..0ad2cb43 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -268,7 +268,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macro.modules %files -%_rpmconfigdir/macro.modules +%_rpmconfigdir/macros.d/macro.modules From 3f9e7a7f5f7e70ac9c6c47ae1f32f7cff518b6e4 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 22:30:54 -0400 Subject: [PATCH 101/106] A convenience method. --- rida/database.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rida/database.py b/rida/database.py index 88a8ce48..3e16782a 100644 --- a/rida/database.py +++ b/rida/database.py @@ -38,6 +38,8 @@ from sqlalchemy.orm import ( ) from sqlalchemy.ext.declarative import declarative_base +import modulemd as _modulemd + import rida.messaging import logging @@ -137,6 +139,14 @@ class ModuleBuild(Base): module = relationship('Module', backref='module_builds', lazy=False) + def mmd(self): + mmd = _modulemd.ModuleMetadata() + try: + mmd.loads(self.modulemd) + except: + raise ValueError("Invalid modulemd") + return mmd + @validates('state') def validate_state(self, key, field): if field in BUILD_STATES.values(): From 874da0ecb8653aeb162d9500857fcbb196638f96 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 22:31:05 -0400 Subject: [PATCH 102/106] Be strict about this. Better to fail early here than to submit a build, wait 15 minutes, only to find out it fails. --- rida/scheduler/handlers/modules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index 7a82f2a4..3c7582b8 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -70,8 +70,7 @@ def wait(config, session, msg): log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag - - dependencies = rida.pdc.get_module_build_dependencies(pdc_session, module_info) + dependencies = rida.pdc.get_module_build_dependencies(pdc_session, module_info, strict=True) builder = rida.builder.KojiModuleBuilder(build.name, config, tag_name=tag) build.buildroot_task_id = builder.buildroot_prep() log.debug("Adding dependencies %s into buildroot for module %s" % (dependencies, module_info)) From 23308474f2bb69d4fe2b6e75c5716de9c9ccf362 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 23:57:54 -0400 Subject: [PATCH 103/106] The filename matters here. --- rida/builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 0ad2cb43..48dad954 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -263,12 +263,12 @@ It should NEVER be installed on any system as it will really mess up %install mkdir -p %buildroot/%_rpmconfigdir/macros.d 2>/dev/null |: -echo %%dist %dist > %buildroot/%_rpmconfigdir/macros.d/macro.modules -chmod 644 %buildroot/%_rpmconfigdir/macros.d/macro.modules +echo %%dist %dist > %buildroot/%_rpmconfigdir/macros.d/macros.modules +chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules %files -%_rpmconfigdir/macros.d/macro.modules +%_rpmconfigdir/macros.d/macros.modules From 074267233c7baaa63aa8fdb1179bd4f8b055e7ca Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 23 Jul 2016 23:57:59 -0400 Subject: [PATCH 104/106] We want our dist-tag hack in both srpm-build and build. --- rida/builder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 48dad954..578d8175 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -384,8 +384,7 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules for nvr in artifacts: self.koji_session.tagBuild(dest_tag, nvr, force=True) if install: - # we usually want just srpm-build - for group in ('srpm-build',): + for group in ('srpm-build', 'build'): pkg_info = kobo.rpmlib.parse_nvr(nvr) log.info("%r adding %s to group %s" % (self, pkg_info['name'], group)) self.koji_session.groupPackageListAdd(dest_tag, group, pkg_info['name']) From cb096f461fe8390ececfc2add66a2205c0eab69b Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 24 Jul 2016 00:52:14 -0400 Subject: [PATCH 105/106] s/task/task_id/ --- rida.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida.py b/rida.py index f62373fb..c8d6f162 100755 --- a/rida.py +++ b/rida.py @@ -168,7 +168,7 @@ def query_build(id): if module.state != "init": for build in db.session.query(rida.database.ComponentBuild).filter_by(module_id=id).all(): tasks[build.format + "/" + build.package] = \ - str(build.task) + "/" + build.state + str(build.task_id) + "/" + build.state return flask.jsonify({ "id": module.id, "state": module.state, From 1b314d2e30e8745488fa23d1828d9eeb0fe6f947 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 24 Jul 2016 00:52:28 -0400 Subject: [PATCH 106/106] New gitref for testmodule. --- submit-build.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submit-build.json b/submit-build.json index 8233168b..96098098 100644 --- a/submit-build.json +++ b/submit-build.json @@ -1,3 +1,3 @@ { - "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#d0c328b61a1b842bb5c400fc8eeb588af7746d2d" + "scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#48932b90de214d9d13feefbd35246a81b6cb8d49" }