diff --git a/rida/messaging.py b/rida/messaging.py index 44b2a859..51e16175 100644 --- a/rida/messaging.py +++ b/rida/messaging.py @@ -111,8 +111,6 @@ class BaseMessage(object): if msg_obj: return msg_obj - log.debug('Skipping unrecognized message with the topic "{0}"' - .format(topic)) return None diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py index f7917a22..41e073a7 100644 --- a/rida/scheduler/handlers/modules.py +++ b/rida/scheduler/handlers/modules.py @@ -106,7 +106,7 @@ def wait(config, session, msg): dependencies, tag = _get_deps_and_tag() except ValueError: log.exception("Failed to get module info from PDC. Max retries reached.") - build.transition(config, state="build") # Wait for the buildroot to be ready.a + build.transition(config, state="failed") session.commit() raise diff --git a/rida/views.py b/rida/views.py index 076dae82..54fcc0ab 100644 --- a/rida/views.py +++ b/rida/views.py @@ -30,11 +30,9 @@ This is the implementation of the orchestrator's public RESTful API. from flask import request, jsonify from flask.views import MethodView import json -import logging import modulemd import os import rida.auth -import rida.logger import rida.scm import shutil import tempfile @@ -85,18 +83,22 @@ class ModuleBuildAPI(MethodView): try: r = json.loads(request.get_data().decode("utf-8")) except: + log.error('Invalid JSON submitted') raise ValidationError('Invalid JSON submitted') if "scmurl" not in r: + log.error('Missing scmurl') raise ValidationError('Missing scmurl') url = r["scmurl"] if not any(url.startswith(prefix) for prefix in conf.scmurls): + log.error('The submitted scmurl is not allowed') raise Unauthorized("The submitted scmurl is not allowed") yaml = "" td = None try: + log.debug('Verifying modulemd') td = tempfile.mkdtemp() scm = rida.scm.SCM(url, conf.scmurls) cod = scm.checkout(td) @@ -117,23 +119,40 @@ class ModuleBuildAPI(MethodView): try: mmd.loads(yaml) except: + log.error('Invalid modulemd') raise UnprocessableEntity('Invalid modulemd') - if models.ModuleBuild.query.filter_by(name=mmd.name, + module = models.ModuleBuild.query.filter_by(name=mmd.name, version=mmd.version, - release=mmd.release).first(): - raise Conflict('Module already exists') - - module = models.ModuleBuild.create( - db.session, - conf, - name=mmd.name, - version=mmd.version, - release=mmd.release, - modulemd=yaml, - scmurl=url, - username=username - ) + release=mmd.release).first() + if module: + log.debug('Checking whether module build already exist.') + # TODO: make this configurable, we might want to allow + # resubmitting any stuck build on DEV no matter the state + if module.state not in (models.BUILD_STATES['failed']): + log.error('Module (state=%s) already exists. ' + 'Only new or failed builds are allowed.' + % module.state) + raise Conflict('Module (state=%s) already exists. ' + 'Only new or failed builds are allowed.' + % module.state) + log.debug('Resuming existing module build %r' % module) + module.username = username + module.transition(conf, models.BUILD_STATES["init"]) + log.info("Resumed existing module build in previous state %s" + % module.state) + else: + log.debug('Creating new module build') + module = models.ModuleBuild.create( + db.session, + conf, + name=mmd.name, + version=mmd.version, + release=mmd.release, + modulemd=yaml, + scmurl=url, + username=username + ) # List of (pkg_name, git_url) tuples to be used to check # the availability of git URLs paralelly later. @@ -166,6 +185,7 @@ class ModuleBuildAPI(MethodView): full_url = pkg["repository"] + "?#" + pkg["commit"] full_urls.append((pkgname, full_url)) + log.debug("Checking scm urls") # Checks the availability of SCM urls. pool = ThreadPool(10) err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0]) @@ -178,20 +198,29 @@ class ModuleBuildAPI(MethodView): for pkgname, pkg in mmd.components.rpms.packages.items(): full_url = pkg["repository"] + "?#" + pkg["commit"] - build = models.ComponentBuild( - module_id=module.id, - package=pkgname, - format="rpms", - scmurl=full_url, - ) - db.session.add(build) + existing_build = models.ComponentBuild.query.filter_by( + module_id=module.id, package=pkgname).first() + if (existing_build + and existing_build.state != models.BUILD_STATES['done']): + existing_build.state = models.BUILD_STATES['init'] + db.session.add(existing_build) + else: + # XXX: what about components that were present in previous + # builds but are gone now (component reduction)? + build = models.ComponentBuild( + module_id=module.id, + package=pkgname, + format="rpms", + scmurl=full_url, + ) + db.session.add(build) module.modulemd = mmd.dumps() module.transition(conf, models.BUILD_STATES["wait"]) db.session.add(module) db.session.commit() - logging.info("%s submitted build of %s-%s-%s", username, mmd.name, - mmd.version, mmd.release) + log.info("%s submitted build of %s-%s-%s", username, mmd.name, + mmd.version, mmd.release) return jsonify(module.json()), 201