From 1c0715662a930b027b225f8b1a19fb002f25af47 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Sun, 19 Jan 2020 10:30:03 +0800 Subject: [PATCH] Cleanup module_build_service/__init__.py Most of the code are moved to dedicated subpackages, but some others can't due to the cycle dependencies. Signed-off-by: Chenxiong Qi --- module_build_service/__init__.py | 100 ++---------------- .../builder/KojiContentGenerator.py | 3 +- .../builder/KojiModuleBuilder.py | 3 +- .../builder/MockModuleBuilder.py | 4 +- module_build_service/builder/base.py | 3 +- module_build_service/builder/utils.py | 3 +- module_build_service/common/__init__.py | 12 +++ module_build_service/common/config.py | 3 + module_build_service/common/errors.py | 8 -- module_build_service/common/koji.py | 2 +- module_build_service/common/messaging.py | 2 +- module_build_service/common/models.py | 3 +- module_build_service/common/modulemd.py | 8 ++ module_build_service/common/resolve.py | 2 +- module_build_service/common/retry.py | 2 +- module_build_service/common/scm.py | 2 +- module_build_service/common/submit.py | 2 +- module_build_service/common/utils.py | 3 +- module_build_service/manage.py | 18 +++- .../526fb7d445f7_module_buildrequires.py | 3 +- .../6d503efcd2b8_virtual_streams_table.py | 3 +- .../708ac8950f55_set_from_mmd_context.py | 2 +- .../versions/9ca1c166f426_contexts.py | 2 +- .../migrations/versions/a87264eeb49f_.py | 2 +- .../caeae7a4f537_ref_build_context.py | 2 +- module_build_service/resolver/DBResolver.py | 3 +- module_build_service/resolver/KojiResolver.py | 3 +- module_build_service/resolver/MBSResolver.py | 2 +- module_build_service/resolver/base.py | 5 +- module_build_service/scheduler/__init__.py | 15 +++ module_build_service/scheduler/batches.py | 3 +- module_build_service/scheduler/consumer.py | 3 +- module_build_service/scheduler/db_session.py | 7 +- .../scheduler/default_modules.py | 4 +- module_build_service/scheduler/events.py | 2 +- module_build_service/scheduler/greenwave.py | 2 +- .../scheduler/handlers/components.py | 5 +- .../scheduler/handlers/greenwave.py | 4 +- .../scheduler/handlers/modules.py | 4 +- .../scheduler/handlers/repos.py | 5 +- .../scheduler/handlers/tags.py | 5 +- module_build_service/scheduler/parser.py | 2 +- module_build_service/scheduler/producer.py | 4 +- module_build_service/scheduler/reuse.py | 3 +- module_build_service/scheduler/route.py | 3 +- module_build_service/scheduler/submit.py | 6 +- module_build_service/scheduler/ursine.py | 2 +- module_build_service/web/auth.py | 3 +- module_build_service/web/backports.py | 6 +- module_build_service/web/mmd_resolver.py | 11 +- module_build_service/web/mse.py | 4 +- module_build_service/web/submit.py | 4 +- module_build_service/web/utils.py | 4 +- module_build_service/web/views.py | 56 +++++++++- tests/__init__.py | 3 +- tests/conftest.py | 4 +- tests/test_build/test_build.py | 4 +- tests/test_builder/test_builder_utils.py | 2 +- tests/test_builder/test_content_generator.py | 4 +- tests/test_builder/test_koji.py | 3 +- tests/test_builder/test_mock.py | 10 +- tests/test_common/test_config.py | 2 +- tests/test_common/test_logger.py | 3 +- tests/test_common/test_models/test_models.py | 2 +- tests/test_common/test_monitor.py | 4 +- tests/test_common/test_resolve.py | 2 +- tests/test_resolver/test_db.py | 3 +- tests/test_resolver/test_koji.py | 2 +- tests/test_resolver/test_local.py | 2 +- tests/test_resolver/test_mbs.py | 3 +- tests/test_scheduler/test_batches.py | 2 +- .../test_scheduler/test_celery_route_task.py | 3 +- tests/test_scheduler/test_db_session.py | 2 +- tests/test_scheduler/test_default_modules.py | 2 +- .../test_scheduler/test_greenwave_handler.py | 2 +- tests/test_scheduler/test_module_init.py | 2 +- tests/test_scheduler/test_module_wait.py | 3 +- tests/test_scheduler/test_poller.py | 2 +- tests/test_scheduler/test_reuse.py | 2 +- tests/test_scheduler/test_submit.py | 4 +- tests/test_scheduler/test_ursine.py | 2 +- 81 files changed, 235 insertions(+), 221 deletions(-) create mode 100644 module_build_service/common/modulemd.py diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index 3728231f..292ffa67 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -19,22 +19,14 @@ for a number of tasks: """ from __future__ import absolute_import -from logging import getLogger -from celery import Celery -import gi # noqa -gi.require_version("Modulemd", "2.0") # noqa -from gi.repository import Modulemd # noqa +import pkg_resources + from flask import Flask, has_app_context, url_for from flask_sqlalchemy import SQLAlchemy -import pkg_resources from sqlalchemy.pool import StaticPool -from module_build_service.common.config import init_config -from module_build_service.common.errors import ( - ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound, - Forbidden, json_error) -from module_build_service.common.logger import init_logging, ModuleBuildLogs, level_flags, MBSLogger +from module_build_service.common.config import config_section from module_build_service.web.proxy import ReverseProxy try: @@ -43,22 +35,10 @@ except pkg_resources.DistributionNotFound: version = "unknown" api_version = 2 -conf, config_section = init_config() app = Flask(__name__) app.wsgi_app = ReverseProxy(app.wsgi_app) app.config.from_object(config_section) -celery_app = Celery("module-build-service") -# Convert config names specific for Celery like this: -# celery_broker_url -> broker_url -celery_configs = { - name[7:]: getattr(conf, name) - for name in dir(conf) if name.startswith("celery_") -} -# Only allow a single process so that tasks are always serial per worker -celery_configs["worker_concurrency"] = 1 -celery_app.conf.update(**celery_configs) - class MBSSQLAlchemy(SQLAlchemy): """ @@ -83,73 +63,6 @@ class MBSSQLAlchemy(SQLAlchemy): db = MBSSQLAlchemy(app) -def create_app(debug=False, verbose=False, quiet=False): - # logging (intended for flask-script, see manage.py) - log = getLogger(__name__) - if debug: - log.setLevel(level_flags["debug"]) - elif verbose: - log.setLevel(level_flags["verbose"]) - elif quiet: - log.setLevel(level_flags["quiet"]) - - return app - - -def load_views(): - from module_build_service.web import views - - assert views - - -@app.errorhandler(ValidationError) -def validationerror_error(e): - """Flask error handler for ValidationError exceptions""" - return json_error(400, "Bad Request", str(e)) - - -@app.errorhandler(Unauthorized) -def unauthorized_error(e): - """Flask error handler for NotAuthorized exceptions""" - return json_error(401, "Unauthorized", str(e)) - - -@app.errorhandler(Forbidden) -def forbidden_error(e): - """Flask error handler for Forbidden exceptions""" - return json_error(403, "Forbidden", str(e)) - - -@app.errorhandler(RuntimeError) -def runtimeerror_error(e): - """Flask error handler for RuntimeError exceptions""" - log.exception("RuntimeError exception raised") - return json_error(500, "Internal Server Error", str(e)) - - -@app.errorhandler(UnprocessableEntity) -def unprocessableentity_error(e): - """Flask error handler for UnprocessableEntity exceptions""" - return json_error(422, "Unprocessable Entity", str(e)) - - -@app.errorhandler(Conflict) -def conflict_error(e): - """Flask error handler for Conflict exceptions""" - return json_error(409, "Conflict", str(e)) - - -@app.errorhandler(NotFound) -def notfound_error(e): - """Flask error handler for Conflict exceptions""" - return json_error(404, "Not Found", str(e)) - - -init_logging(conf) -log = MBSLogger() -build_logs = ModuleBuildLogs(conf.build_logs_dir, conf.build_logs_name_format, conf.log_level) - - def get_url_for(*args, **kwargs): """ flask.url_for wrapper which creates the app_context on-the-fly. @@ -161,6 +74,7 @@ def get_url_for(*args, **kwargs): # system as the web views. app.config["SERVER_NAME"] = "localhost" with app.app_context(): + from module_build_service.common import log log.debug( "WARNING: get_url_for() has been called without the Flask " "app_context. That can lead to SQLAlchemy errors caused by " @@ -169,4 +83,10 @@ def get_url_for(*args, **kwargs): return url_for(*args, **kwargs) +def load_views(): + from module_build_service.web import views + + assert views + + load_views() diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 321646ed..d7f65dd7 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -21,7 +21,8 @@ import koji import pungi.arch from six import text_type -from module_build_service import conf, log, build_logs, Modulemd +from module_build_service.common.modulemd import Modulemd +from module_build_service.common import conf, log, build_logs from module_build_service.common.koji import get_session, koji_retrying_multicall_map from module_build_service.common.scm import SCM from module_build_service.common.utils import load_mmd, mmd_to_str, to_text_type diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index 3a4858d9..9c89441e 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -22,11 +22,10 @@ import koji import kobo.rpmlib from OpenSSL.SSL import SysCallError -from module_build_service import log, conf from module_build_service.builder import GenericBuilder from module_build_service.builder.KojiContentGenerator import KojiContentGenerator from module_build_service.builder.utils import execute_cmd, get_rpm_release, validate_koji_tag -from module_build_service.common import models +from module_build_service.common import log, conf, models from module_build_service.common.koji import ( get_session, koji_multicall_map, koji_retrying_multicall_map, ) diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 4a7810c7..70e3e955 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -13,7 +13,6 @@ import koji import kobo.rpmlib import platform -from module_build_service import conf, log, Modulemd from module_build_service.builder import GenericBuilder from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder from module_build_service.builder.utils import ( @@ -23,8 +22,9 @@ from module_build_service.builder.utils import ( get_koji_config, validate_koji_tag, ) -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.common.koji import get_session +from module_build_service.common.modulemd import Modulemd from module_build_service.common.utils import import_mmd, load_mmd_file, mmd_to_str from module_build_service.scheduler import events from module_build_service.scheduler.db_session import db_session diff --git a/module_build_service/builder/base.py b/module_build_service/builder/base.py index 13443f5d..d122823b 100644 --- a/module_build_service/builder/base.py +++ b/module_build_service/builder/base.py @@ -9,8 +9,7 @@ import dogpile.cache from requests.exceptions import ConnectionError import six -from module_build_service import conf, log -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.common.models import BUILD_STATES from module_build_service.common.retry import retry from module_build_service.resolver import GenericResolver diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py index d987f3fc..fe5d1003 100644 --- a/module_build_service/builder/utils.py +++ b/module_build_service/builder/utils.py @@ -13,8 +13,7 @@ import subprocess import munch import requests -from module_build_service import conf, log -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.common.errors import ValidationError, ProgrammingError diff --git a/module_build_service/common/__init__.py b/module_build_service/common/__init__.py index e69de29b..b0612ea1 100644 --- a/module_build_service/common/__init__.py +++ b/module_build_service/common/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT + +from module_build_service.common.config import conf +from module_build_service.common import logger + +__all__ = ('conf', 'log', 'build_logs') + +logger.init_logging(conf) +log = logger.MBSLogger() +build_logs = logger.ModuleBuildLogs( + conf.build_logs_dir, conf.build_logs_name_format, conf.log_level) diff --git a/module_build_service/common/config.py b/module_build_service/common/config.py index a77c79c8..77c32dd6 100644 --- a/module_build_service/common/config.py +++ b/module_build_service/common/config.py @@ -989,3 +989,6 @@ class Config(object): if i < 1: raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1") self._num_threads_for_build_submissions = i + + +conf, config_section = init_config() diff --git a/module_build_service/common/errors.py b/module_build_service/common/errors.py index a49ffe69..7c370a88 100644 --- a/module_build_service/common/errors.py +++ b/module_build_service/common/errors.py @@ -4,8 +4,6 @@ from __future__ import absolute_import -from flask import jsonify - class ValidationError(ValueError): pass @@ -43,11 +41,5 @@ class GreenwaveError(RuntimeError): pass -def json_error(status, error, message): - response = jsonify({"status": status, "error": error, "message": message}) - response.status_code = status - return response - - class IgnoreMessage(Exception): """Raise if message received from message bus should be ignored""" diff --git a/module_build_service/common/koji.py b/module_build_service/common/koji.py index cf779d93..d7f6793b 100644 --- a/module_build_service/common/koji.py +++ b/module_build_service/common/koji.py @@ -7,7 +7,7 @@ import koji import munch import six.moves.xmlrpc_client as xmlrpclib -from module_build_service import log +from module_build_service.common import log from module_build_service.common.retry import retry from module_build_service.common.errors import ProgrammingError diff --git a/module_build_service/common/messaging.py b/module_build_service/common/messaging.py index e5f22b5c..82c71ff4 100644 --- a/module_build_service/common/messaging.py +++ b/module_build_service/common/messaging.py @@ -7,7 +7,7 @@ from __future__ import absolute_import import pkg_resources import six.moves.queue -from module_build_service import conf, log +from module_build_service.common import conf, log from module_build_service.scheduler.parser import FedmsgMessageParser # A queue containing message body that should be sent after diff --git a/module_build_service/common/models.py b/module_build_service/common/models.py index 0894cc32..5a733fa1 100644 --- a/module_build_service/common/models.py +++ b/module_build_service/common/models.py @@ -19,7 +19,8 @@ from sqlalchemy.orm import validates, load_only from sqlalchemy.schema import Index -from module_build_service import db, log, get_url_for, conf +from module_build_service import db, get_url_for +from module_build_service.common import conf, log from module_build_service.common.errors import UnprocessableEntity from module_build_service.common.messaging import module_build_state_change_out_queue from module_build_service.common.messaging import notify_on_module_state_change diff --git a/module_build_service/common/modulemd.py b/module_build_service/common/modulemd.py new file mode 100644 index 00000000..26059234 --- /dev/null +++ b/module_build_service/common/modulemd.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT + +import gi +gi.require_version("Modulemd", "2.0") +from gi.repository import Modulemd # noqa + +__all__ = ['Modulemd'] diff --git a/module_build_service/common/resolve.py b/module_build_service/common/resolve.py index 363b31af..81c1b0c5 100644 --- a/module_build_service/common/resolve.py +++ b/module_build_service/common/resolve.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT from __future__ import absolute_import -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.common.errors import StreamAmbigous from module_build_service.resolver import GenericResolver diff --git a/module_build_service/common/retry.py b/module_build_service/common/retry.py index 9e1f8f73..a8728e13 100644 --- a/module_build_service/common/retry.py +++ b/module_build_service/common/retry.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import functools import time -from module_build_service import conf, log +from module_build_service.common import conf, log def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception): diff --git a/module_build_service/common/scm.py b/module_build_service/common/scm.py index f0614fcf..c8b1d79a 100644 --- a/module_build_service/common/scm.py +++ b/module_build_service/common/scm.py @@ -10,7 +10,7 @@ import re import shutil import tempfile -from module_build_service import log, conf +from module_build_service.common import log, conf from module_build_service.common.errors import ( Forbidden, ValidationError, diff --git a/module_build_service/common/submit.py b/module_build_service/common/submit.py index a182637d..b0b78494 100644 --- a/module_build_service/common/submit.py +++ b/module_build_service/common/submit.py @@ -7,7 +7,7 @@ import tempfile import requests import module_build_service.common.scm -from module_build_service import conf, log +from module_build_service.common import conf, log from module_build_service.common.errors import ValidationError from module_build_service.common.utils import load_mmd_file diff --git a/module_build_service/common/utils.py b/module_build_service/common/utils.py index 1fea79ec..b80b829d 100644 --- a/module_build_service/common/utils.py +++ b/module_build_service/common/utils.py @@ -8,8 +8,9 @@ import os from gi.repository.GLib import Error as ModuleMDError from six import string_types, text_type -from module_build_service import conf, log, Modulemd +from module_build_service.common import conf, log from module_build_service.common.errors import UnprocessableEntity +from module_build_service.common.modulemd import Modulemd def to_text_type(s): diff --git a/module_build_service/manage.py b/module_build_service/manage.py index d4d5425d..925b5031 100755 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -11,12 +11,13 @@ import flask_migrate from flask_script import Manager, prompt_bool from werkzeug.datastructures import FileStorage -from module_build_service import app, conf, create_app, db +from module_build_service import app, db from module_build_service.builder.MockModuleBuilder import ( import_builds_from_local_dnf_repos, load_local_builds ) -from module_build_service.common import models +from module_build_service.common import conf, models from module_build_service.common.errors import StreamAmbigous +from module_build_service.common.logger import level_flags from module_build_service.common.utils import load_mmd_file, import_mmd import module_build_service.scheduler.consumer from module_build_service.scheduler.db_session import db_session @@ -24,6 +25,19 @@ import module_build_service.scheduler.local from module_build_service.web.submit import submit_module_build_from_yaml +def create_app(debug=False, verbose=False, quiet=False): + # logging (intended for flask-script, see manage.py) + log = logging.getLogger(__name__) + if debug: + log.setLevel(level_flags["debug"]) + elif verbose: + log.setLevel(level_flags["verbose"]) + elif quiet: + log.setLevel(level_flags["quiet"]) + + return app + + manager = Manager(create_app) help_args = ("-?", "--help") manager.help_args = help_args diff --git a/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py b/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py index 2d24dc3c..7d02594e 100644 --- a/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py +++ b/module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py @@ -14,8 +14,9 @@ from alembic import op import sqlalchemy as sa # Data migration imports -from module_build_service import Modulemd, conf +from module_build_service.common import conf from module_build_service.common.models import ModuleBuild +from module_build_service.common.modulemd import Modulemd # Data migration tables mb = sa.Table( diff --git a/module_build_service/migrations/versions/6d503efcd2b8_virtual_streams_table.py b/module_build_service/migrations/versions/6d503efcd2b8_virtual_streams_table.py index f5763b6f..c041ff53 100644 --- a/module_build_service/migrations/versions/6d503efcd2b8_virtual_streams_table.py +++ b/module_build_service/migrations/versions/6d503efcd2b8_virtual_streams_table.py @@ -16,7 +16,8 @@ from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base # Data migration imports -from module_build_service import Modulemd, conf +from module_build_service.common import conf +from module_build_service.common.modulemd import Modulemd Base = declarative_base() diff --git a/module_build_service/migrations/versions/708ac8950f55_set_from_mmd_context.py b/module_build_service/migrations/versions/708ac8950f55_set_from_mmd_context.py index f7923420..0d80965b 100644 --- a/module_build_service/migrations/versions/708ac8950f55_set_from_mmd_context.py +++ b/module_build_service/migrations/versions/708ac8950f55_set_from_mmd_context.py @@ -14,8 +14,8 @@ from alembic import op import sqlalchemy as sa # Data migration imports -from module_build_service import Modulemd import hashlib +from module_build_service.common.modulemd import Modulemd modulebuild = sa.Table( diff --git a/module_build_service/migrations/versions/9ca1c166f426_contexts.py b/module_build_service/migrations/versions/9ca1c166f426_contexts.py index 81d81208..7d303480 100644 --- a/module_build_service/migrations/versions/9ca1c166f426_contexts.py +++ b/module_build_service/migrations/versions/9ca1c166f426_contexts.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa # Data migration imports -from module_build_service import Modulemd +from module_build_service.common.modulemd import Modulemd import hashlib import json from collections import OrderedDict diff --git a/module_build_service/migrations/versions/a87264eeb49f_.py b/module_build_service/migrations/versions/a87264eeb49f_.py index aae8a9bc..78f91fd5 100644 --- a/module_build_service/migrations/versions/a87264eeb49f_.py +++ b/module_build_service/migrations/versions/a87264eeb49f_.py @@ -12,8 +12,8 @@ down_revision = "1817e62719f9" from alembic import op import sqlalchemy as sa -from module_build_service import Modulemd from module_build_service.common.models import ModuleBuild +from module_build_service.common.modulemd import Modulemd modulebuild = sa.Table( diff --git a/module_build_service/migrations/versions/caeae7a4f537_ref_build_context.py b/module_build_service/migrations/versions/caeae7a4f537_ref_build_context.py index 2f047fb7..ffc2e25e 100644 --- a/module_build_service/migrations/versions/caeae7a4f537_ref_build_context.py +++ b/module_build_service/migrations/versions/caeae7a4f537_ref_build_context.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa # Data migration imports -from module_build_service import Modulemd +from module_build_service.common.modulemd import Modulemd import hashlib import json from collections import OrderedDict diff --git a/module_build_service/resolver/DBResolver.py b/module_build_service/resolver/DBResolver.py index b1a3acd4..a0aa433d 100644 --- a/module_build_service/resolver/DBResolver.py +++ b/module_build_service/resolver/DBResolver.py @@ -5,8 +5,7 @@ from __future__ import absolute_import import sqlalchemy from sqlalchemy.orm import aliased -from module_build_service import log -from module_build_service.common import models +from module_build_service.common import log, models from module_build_service.common.errors import UnprocessableEntity from module_build_service.common.utils import load_mmd from module_build_service.resolver.base import GenericResolver diff --git a/module_build_service/resolver/KojiResolver.py b/module_build_service/resolver/KojiResolver.py index 919e319d..ebea974f 100644 --- a/module_build_service/resolver/KojiResolver.py +++ b/module_build_service/resolver/KojiResolver.py @@ -3,8 +3,7 @@ from __future__ import absolute_import from itertools import groupby -from module_build_service import conf, log -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.common.koji import get_session, koji_multicall_map from module_build_service.resolver.DBResolver import DBResolver diff --git a/module_build_service/resolver/MBSResolver.py b/module_build_service/resolver/MBSResolver.py index 98bbe7cc..3cd8dce9 100644 --- a/module_build_service/resolver/MBSResolver.py +++ b/module_build_service/resolver/MBSResolver.py @@ -7,8 +7,8 @@ import logging import kobo.rpmlib -from module_build_service import conf from module_build_service.common import models +from module_build_service.common.config import conf from module_build_service.common.errors import UnprocessableEntity from module_build_service.common.request_utils import requests_session from module_build_service.common.utils import load_mmd, import_mmd diff --git a/module_build_service/resolver/base.py b/module_build_service/resolver/base.py index aeb40a6b..776ff78d 100644 --- a/module_build_service/resolver/base.py +++ b/module_build_service/resolver/base.py @@ -7,8 +7,7 @@ from __future__ import absolute_import from abc import ABCMeta, abstractmethod import six -from module_build_service import conf -import module_build_service.common.config as cfg +from module_build_service.common.config import conf, SUPPORTED_RESOLVERS class GenericResolver(six.with_metaclass(ABCMeta)): @@ -16,7 +15,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)): External Api for resolvers """ - _resolvers = cfg.SUPPORTED_RESOLVERS + _resolvers = SUPPORTED_RESOLVERS # Resolver name. Each subclass of GenericResolver must set its own name. backend = "generic" diff --git a/module_build_service/scheduler/__init__.py b/module_build_service/scheduler/__init__.py index 01d69e8d..ccb59420 100644 --- a/module_build_service/scheduler/__init__.py +++ b/module_build_service/scheduler/__init__.py @@ -1,3 +1,18 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT """ This is a sub-module for backend/scheduler functionality. """ + +from celery import Celery + +from module_build_service.common.config import conf + +celery_app = Celery("module-build-service") +# Convert config names specific for Celery like this: +# celery_broker_url -> broker_url +celery_configs = { + name[7:]: getattr(conf, name) + for name in dir(conf) if name.startswith("celery_") +} +# Only allow a single process so that tasks are always serial per worker +celery_configs["worker_concurrency"] = 1 +celery_app.conf.update(**celery_configs) diff --git a/module_build_service/scheduler/batches.py b/module_build_service/scheduler/batches.py index 1994b4ab..ef8f4e96 100644 --- a/module_build_service/scheduler/batches.py +++ b/module_build_service/scheduler/batches.py @@ -4,8 +4,7 @@ from __future__ import absolute_import import concurrent.futures import threading -from module_build_service import conf, log -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.scheduler import events from module_build_service.scheduler.db_session import db_session from module_build_service.scheduler.reuse import get_reusable_components, reuse_component diff --git a/module_build_service/scheduler/consumer.py b/module_build_service/scheduler/consumer.py index 7ba2ad8b..c242666f 100644 --- a/module_build_service/scheduler/consumer.py +++ b/module_build_service/scheduler/consumer.py @@ -20,8 +20,7 @@ import fedmsg.consumers import moksha.hub import sqlalchemy.exc -from module_build_service import log, conf -from module_build_service.common import models +from module_build_service.common import log, conf, models from module_build_service.common.errors import IgnoreMessage import module_build_service.common.messaging from module_build_service.common.messaging import default_messaging_backend diff --git a/module_build_service/scheduler/db_session.py b/module_build_service/scheduler/db_session.py index 1e2890f6..580aed44 100644 --- a/module_build_service/scheduler/db_session.py +++ b/module_build_service/scheduler/db_session.py @@ -6,9 +6,10 @@ import sqlalchemy.event from sqlalchemy.pool import NullPool from sqlalchemy.orm import scoped_session, sessionmaker -from module_build_service import conf -from module_build_service.common.models import send_message_after_module_build_state_change -from module_build_service.common.models import session_before_commit_handlers +from module_build_service.common.config import conf +from module_build_service.common.models import ( + session_before_commit_handlers, send_message_after_module_build_state_change +) __all__ = ("db_session",) diff --git a/module_build_service/scheduler/default_modules.py b/module_build_service/scheduler/default_modules.py index 62877454..40612012 100644 --- a/module_build_service/scheduler/default_modules.py +++ b/module_build_service/scheduler/default_modules.py @@ -11,10 +11,10 @@ import kobo.rpmlib import koji import six.moves.xmlrpc_client as xmlrpclib -from module_build_service import conf, log, Modulemd -from module_build_service.common import models, scm +from module_build_service.common import conf, log, models, scm from module_build_service.common.errors import UnprocessableEntity from module_build_service.common.koji import get_session, koji_retrying_multicall_map +from module_build_service.common.modulemd import Modulemd from module_build_service.common.resolve import ( expand_single_mse_streams, get_compatible_base_module_mmds ) diff --git a/module_build_service/scheduler/events.py b/module_build_service/scheduler/events.py index 4a176724..f0060b2c 100644 --- a/module_build_service/scheduler/events.py +++ b/module_build_service/scheduler/events.py @@ -17,7 +17,7 @@ from functools import wraps import sched import time -from module_build_service import log +from module_build_service.common import log KOJI_BUILD_CHANGE = "koji_build_change" diff --git a/module_build_service/scheduler/greenwave.py b/module_build_service/scheduler/greenwave.py index 1b00d5e4..9f1b770c 100644 --- a/module_build_service/scheduler/greenwave.py +++ b/module_build_service/scheduler/greenwave.py @@ -6,7 +6,7 @@ import json import requests -from module_build_service import log, conf +from module_build_service.common import log, conf from module_build_service.common.errors import GreenwaveError diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index 6dc2ce34..3a46c726 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -7,12 +7,11 @@ import logging import koji -from module_build_service import celery_app, conf, log from module_build_service.builder import GenericBuilder -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.common.koji import get_session from module_build_service.common.utils import mmd_to_str -from module_build_service.scheduler import events +from module_build_service.scheduler import celery_app, events from module_build_service.scheduler.batches import continue_batch_build from module_build_service.scheduler.db_session import db_session diff --git a/module_build_service/scheduler/handlers/greenwave.py b/module_build_service/scheduler/handlers/greenwave.py index 36bcf934..e8898c9b 100644 --- a/module_build_service/scheduler/handlers/greenwave.py +++ b/module_build_service/scheduler/handlers/greenwave.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: MIT from __future__ import absolute_import -from module_build_service import celery_app, conf, log +from module_build_service.common import conf, log from module_build_service.common.koji import get_session from module_build_service.common.models import ModuleBuild, BUILD_STATES from module_build_service.scheduler.db_session import db_session -from module_build_service.scheduler import events +from module_build_service.scheduler import celery_app, events def get_corresponding_module_build(nvr): diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 35e44db2..e8fc6253 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -11,11 +11,11 @@ import koji from requests.exceptions import ConnectionError import six.moves.xmlrpc_client as xmlrpclib -from module_build_service import celery_app, conf, log, build_logs from module_build_service.builder import GenericBuilder from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder from module_build_service.builder.utils import get_rpm_release from module_build_service.common import models +from module_build_service.common import build_logs, conf, log from module_build_service.common.errors import UnprocessableEntity, Forbidden, ValidationError from module_build_service.common.utils import mmd_to_str from module_build_service.common.retry import retry @@ -25,7 +25,7 @@ from module_build_service.scheduler.submit import ( record_filtered_rpms, record_module_build_arches ) -from module_build_service.scheduler import events +from module_build_service.scheduler import celery_app, events from module_build_service.scheduler.db_session import db_session from module_build_service.scheduler.default_modules import ( add_default_modules, handle_collisions_with_base_module_rpms) diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py index 829b545b..1d190644 100644 --- a/module_build_service/scheduler/handlers/repos.py +++ b/module_build_service/scheduler/handlers/repos.py @@ -6,10 +6,9 @@ from __future__ import absolute_import from datetime import datetime import logging -from module_build_service import celery_app, conf, log from module_build_service.builder import GenericBuilder -from module_build_service.common import models -from module_build_service.scheduler import events +from module_build_service.common import conf, log, models +from module_build_service.scheduler import celery_app, events from module_build_service.scheduler.batches import start_next_batch_build from module_build_service.scheduler.db_session import db_session diff --git a/module_build_service/scheduler/handlers/tags.py b/module_build_service/scheduler/handlers/tags.py index 2e3eff10..84e31fc5 100644 --- a/module_build_service/scheduler/handlers/tags.py +++ b/module_build_service/scheduler/handlers/tags.py @@ -7,10 +7,9 @@ import logging import koji -from module_build_service import celery_app, conf, log from module_build_service.builder import GenericBuilder -from module_build_service.common import models -from module_build_service.scheduler import events +from module_build_service.common import conf, log, models +from module_build_service.scheduler import celery_app, events from module_build_service.scheduler.db_session import db_session logging.basicConfig(level=logging.DEBUG) diff --git a/module_build_service/scheduler/parser.py b/module_build_service/scheduler/parser.py index 17388675..77859b24 100644 --- a/module_build_service/scheduler/parser.py +++ b/module_build_service/scheduler/parser.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import re -from module_build_service import log +from module_build_service.common import log from module_build_service.common.errors import IgnoreMessage from module_build_service.scheduler import events diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index 72a18027..3e80db6d 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -8,12 +8,12 @@ import koji from sqlalchemy.orm import lazyload, load_only -from module_build_service import celery_app, conf, log +from module_build_service.common import conf, log, models from module_build_service.builder import GenericBuilder -from module_build_service.common import models from module_build_service.common.koji import get_session import module_build_service.scheduler import module_build_service.scheduler.consumer +from module_build_service.scheduler import celery_app from module_build_service.scheduler.consumer import ON_MODULE_CHANGE_HANDLERS from module_build_service.scheduler.batches import ( at_concurrent_component_threshold, diff --git a/module_build_service/scheduler/reuse.py b/module_build_service/scheduler/reuse.py index 061f84bc..57305196 100644 --- a/module_build_service/scheduler/reuse.py +++ b/module_build_service/scheduler/reuse.py @@ -4,8 +4,7 @@ from __future__ import absolute_import import kobo.rpmlib -from module_build_service import log, conf -from module_build_service.common import models +from module_build_service.common import log, conf, models from module_build_service.common.resolve import get_base_module_mmds from module_build_service.resolver import GenericResolver from module_build_service.scheduler import events diff --git a/module_build_service/scheduler/route.py b/module_build_service/scheduler/route.py index 5838a3ec..c74c78c6 100644 --- a/module_build_service/scheduler/route.py +++ b/module_build_service/scheduler/route.py @@ -5,8 +5,7 @@ from __future__ import absolute_import import inspect -from module_build_service import conf, log -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.scheduler.db_session import db_session from module_build_service.scheduler.handlers.greenwave import get_corresponding_module_build diff --git a/module_build_service/scheduler/submit.py b/module_build_service/scheduler/submit.py index 7e954d8d..5f39ed73 100644 --- a/module_build_service/scheduler/submit.py +++ b/module_build_service/scheduler/submit.py @@ -8,10 +8,10 @@ import os import kobo.rpmlib -from module_build_service import conf, log, Modulemd -from module_build_service.common import models -from module_build_service.common.errors import ValidationError, UnprocessableEntity, Forbidden import module_build_service.common.scm +from module_build_service.common import conf, log, models +from module_build_service.common.errors import ValidationError, UnprocessableEntity, Forbidden +from module_build_service.common.modulemd import Modulemd from module_build_service.common.submit import fetch_mmd from module_build_service.common.utils import to_text_type from module_build_service.scheduler.db_session import db_session diff --git a/module_build_service/scheduler/ursine.py b/module_build_service/scheduler/ursine.py index c19fe157..cc890010 100644 --- a/module_build_service/scheduler/ursine.py +++ b/module_build_service/scheduler/ursine.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import re -from module_build_service import conf, log +from module_build_service.common import conf, log from module_build_service.common.koji import get_session from module_build_service.resolver import GenericResolver from module_build_service.scheduler.db_session import db_session diff --git a/module_build_service/web/auth.py b/module_build_service/web/auth.py index 256cd26b..b1690c6a 100644 --- a/module_build_service/web/auth.py +++ b/module_build_service/web/auth.py @@ -10,7 +10,8 @@ from dogpile.cache import make_region from flask import g import requests -from module_build_service import app, log, conf +from module_build_service import app +from module_build_service.common import conf, log from module_build_service.common.errors import Unauthorized, Forbidden diff --git a/module_build_service/web/backports.py b/module_build_service/web/backports.py index f2c8c631..1729e6de 100644 --- a/module_build_service/web/backports.py +++ b/module_build_service/web/backports.py @@ -5,7 +5,7 @@ from __future__ import absolute_import from flask import request from flask.json import dumps -import module_build_service +from module_build_service import app def jsonify(*args, **kwargs): @@ -16,7 +16,7 @@ def jsonify(*args, **kwargs): indent = None separators = (",", ":") - if module_build_service.app.config["JSONIFY_PRETTYPRINT_REGULAR"] and not request.is_xhr: + if app.config["JSONIFY_PRETTYPRINT_REGULAR"] and not request.is_xhr: indent = 2 separators = (", ", ": ") @@ -31,6 +31,6 @@ def jsonify(*args, **kwargs): # Note that we add '\n' to end of response # (see https://github.com/mitsuhiko/flask/pull/1262) - rv = module_build_service.app.response_class( + rv = app.response_class( (dumps(data, indent=indent, separators=separators), "\n"), mimetype="application/json") return rv diff --git a/module_build_service/web/mmd_resolver.py b/module_build_service/web/mmd_resolver.py index a641b03d..2b3be360 100644 --- a/module_build_service/web/mmd_resolver.py +++ b/module_build_service/web/mmd_resolver.py @@ -6,8 +6,7 @@ import itertools import solv -from module_build_service import log, conf -from module_build_service.common.models import ModuleBuild +from module_build_service.common import log, conf, models class MMDResolver(object): @@ -144,7 +143,8 @@ class MMDResolver(object): # In case x.y.z versioning is not used for this base module, do not # use versions solv.Dep. stream_version_str = str( - ModuleBuild.get_stream_version(stream_for_version, right_pad=False)) + models.ModuleBuild.get_stream_version( + stream_for_version, right_pad=False)) if len(stream_version_str) < 5: req_pos = rel_or_dep( req_pos, solv.REL_OR, self.module_dep(name, stream)) @@ -173,7 +173,7 @@ class MMDResolver(object): op = solv.REL_EQ if not exact_versions: op |= solv.REL_GT - version = ModuleBuild.get_stream_version( + version = models.ModuleBuild.get_stream_version( stream_for_version, right_pad=False ) req_pos = rel_or_dep( @@ -223,7 +223,8 @@ class MMDResolver(object): # and so on. We therefore need to convert the stream and version of base module to # integer representation and add "module($name:$stream) = $stream_based_version" # to Provides. - stream_version = ModuleBuild.get_stream_version(mmd.get_stream_name(), right_pad=False) + stream_version = models.ModuleBuild.get_stream_version( + mmd.get_stream_name(), right_pad=False) if stream_version: base_stream_ver = True self.solvable_provides( diff --git a/module_build_service/web/mse.py b/module_build_service/web/mse.py index a800db90..5c53216d 100644 --- a/module_build_service/web/mse.py +++ b/module_build_service/web/mse.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: MIT from __future__ import absolute_import -from module_build_service import log, Modulemd, conf -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.common.errors import StreamAmbigous, UnprocessableEntity +from module_build_service.common.modulemd import Modulemd from module_build_service.common.resolve import expand_single_mse_streams, get_base_module_mmds from module_build_service.common.utils import mmd_to_str from module_build_service.resolver import GenericResolver diff --git a/module_build_service/web/submit.py b/module_build_service/web/submit.py index 96d90362..15a74b11 100644 --- a/module_build_service/web/submit.py +++ b/module_build_service/web/submit.py @@ -11,10 +11,10 @@ import time from gi.repository import GLib import requests -from module_build_service import conf, log, Modulemd -from module_build_service.common import models +from module_build_service.common import conf, log, models from module_build_service.common.errors import Conflict, Forbidden, ValidationError from module_build_service.common.messaging import notify_on_module_state_change +from module_build_service.common.modulemd import Modulemd from module_build_service.common.submit import fetch_mmd from module_build_service.common.utils import load_mmd, mmd_to_str, to_text_type from module_build_service.web.mse import generate_expanded_mmds diff --git a/module_build_service/web/utils.py b/module_build_service/web/utils.py index c27afa43..3de81c9b 100644 --- a/module_build_service/web/utils.py +++ b/module_build_service/web/utils.py @@ -11,8 +11,8 @@ import sqlalchemy from sqlalchemy.orm import aliased from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean -from module_build_service import api_version, conf, db -from module_build_service.common import models +from module_build_service import api_version, db +from module_build_service.common import conf, models from module_build_service.common.errors import ValidationError, NotFound from module_build_service.common.scm import scm_url_schemes diff --git a/module_build_service/web/views.py b/module_build_service/web/views.py index 75252f85..0488a1da 100644 --- a/module_build_service/web/views.py +++ b/module_build_service/web/views.py @@ -14,10 +14,11 @@ from flask.views import MethodView from prometheus_client import generate_latest, CONTENT_TYPE_LATEST from six import string_types -from module_build_service import app, conf, log, db, version, api_version as max_api_version -from module_build_service.common import models +from module_build_service import app, db, version, api_version as max_api_version +from module_build_service.common import conf, log, models from module_build_service.common.errors import ( - ValidationError, Forbidden, NotFound, ProgrammingError + ValidationError, Forbidden, NotFound, ProgrammingError, + Unauthorized, UnprocessableEntity, Conflict ) from module_build_service.common.models import send_message_after_module_build_state_change from module_build_service.common.monitor import registry @@ -569,6 +570,55 @@ def register_api(): register_api() +def json_error(status, error, message): + response = jsonify({"status": status, "error": error, "message": message}) + response.status_code = status + return response + + +@app.errorhandler(ValidationError) +def validationerror_error(e): + """Flask error handler for ValidationError exceptions""" + return json_error(400, "Bad Request", str(e)) + + +@app.errorhandler(Unauthorized) +def unauthorized_error(e): + """Flask error handler for NotAuthorized exceptions""" + return json_error(401, "Unauthorized", str(e)) + + +@app.errorhandler(Forbidden) +def forbidden_error(e): + """Flask error handler for Forbidden exceptions""" + return json_error(403, "Forbidden", str(e)) + + +@app.errorhandler(RuntimeError) +def runtimeerror_error(e): + """Flask error handler for RuntimeError exceptions""" + log.exception("RuntimeError exception raised") + return json_error(500, "Internal Server Error", str(e)) + + +@app.errorhandler(UnprocessableEntity) +def unprocessableentity_error(e): + """Flask error handler for UnprocessableEntity exceptions""" + return json_error(422, "Unprocessable Entity", str(e)) + + +@app.errorhandler(Conflict) +def conflict_error(e): + """Flask error handler for Conflict exceptions""" + return json_error(409, "Conflict", str(e)) + + +@app.errorhandler(NotFound) +def notfound_error(e): + """Flask error handler for Conflict exceptions""" + return json_error(404, "Not Found", str(e)) + + # Ensure the event handler is called on db.session sqlalchemy.event.listen( db.session, "after_commit", send_message_after_module_build_state_change) diff --git a/tests/__init__.py b/tests/__init__.py index 62d18798..a724ac56 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,7 +16,7 @@ from mock import patch from six import string_types import module_build_service -from module_build_service import db, Modulemd +from module_build_service import db from module_build_service.builder.utils import get_rpm_release from module_build_service.common.models import ( BUILD_STATES, @@ -25,6 +25,7 @@ from module_build_service.common.models import ( ModuleBuild, VirtualStream, ) +from module_build_service.common.modulemd import Modulemd from module_build_service.common.utils import load_mmd, import_mmd, mmd_to_str, to_text_type from module_build_service.scheduler.db_session import db_session diff --git a/tests/conftest.py b/tests/conftest.py index 3a1c8f0c..7b660573 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -390,10 +390,10 @@ def cleanup_build_logs(request): """ def _cleanup_build_logs(): - build_ids = list(module_build_service.build_logs.handlers.keys()) + build_ids = list(module_build_service.common.build_logs.handlers.keys()) for build_id in build_ids: mock_build = mock.Mock() mock_build.id = build_id - module_build_service.build_logs.stop(mock_build) + module_build_service.common.build_logs.stop(mock_build) request.addfinalizer(_cleanup_build_logs) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 03f4b73a..8f84bf45 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -21,12 +21,12 @@ import moksha.hub import pytest from werkzeug.datastructures import FileStorage -from module_build_service import app, conf, build_logs, log +from module_build_service import app from module_build_service.builder import GenericBuilder from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder from module_build_service.builder.MockModuleBuilder import load_local_builds from module_build_service.builder.utils import get_rpm_release, validate_koji_tag -from module_build_service.common import models +from module_build_service.common import conf, log, build_logs, models from module_build_service.common.errors import Forbidden from module_build_service.common.utils import load_mmd, import_mmd from module_build_service.scheduler import events diff --git a/tests/test_builder/test_builder_utils.py b/tests/test_builder/test_builder_utils.py index dbad5db4..2ce9e0d4 100644 --- a/tests/test_builder/test_builder_utils.py +++ b/tests/test_builder/test_builder_utils.py @@ -7,9 +7,9 @@ import tempfile from mock import call, MagicMock, Mock, patch, PropertyMock import pytest -from module_build_service import conf from module_build_service.builder import utils from module_build_service.common import models +from module_build_service.common.config import conf from module_build_service.common.errors import ProgrammingError, ValidationError from module_build_service.common.utils import load_mmd, import_mmd, mmd_to_str from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_builder/test_content_generator.py b/tests/test_builder/test_content_generator.py index 1b66de20..2391f237 100644 --- a/tests/test_builder/test_content_generator.py +++ b/tests/test_builder/test_content_generator.py @@ -11,9 +11,9 @@ import koji from mock import patch, Mock, call, mock_open import pytest -from module_build_service import conf, build_logs, Modulemd from module_build_service.builder.KojiContentGenerator import KojiContentGenerator -from module_build_service.common import models +from module_build_service.common import conf, build_logs, models +from module_build_service.common.modulemd import Modulemd from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str from module_build_service.scheduler.db_session import db_session from tests import init_data diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 32ba8bb7..e3b03113 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -11,10 +11,11 @@ import mock from mock import patch, MagicMock import pytest -from module_build_service import conf, Modulemd from module_build_service.builder import GenericBuilder from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder import module_build_service.common.models +from module_build_service.common.config import conf +from module_build_service.common.modulemd import Modulemd from module_build_service.common.utils import mmd_to_str from module_build_service.scheduler import events from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_builder/test_mock.py b/tests/test_builder/test_mock.py index d1015a67..1f0c9d03 100644 --- a/tests/test_builder/test_mock.py +++ b/tests/test_builder/test_mock.py @@ -11,7 +11,7 @@ import koji import mock import pytest -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.builder.MockModuleBuilder import ( import_fake_base_module, import_builds_from_local_dnf_repos, @@ -122,7 +122,7 @@ class TestMockModuleBuilder: return module - @mock.patch("module_build_service.conf.system", new="mock") + @mock.patch("module_build_service.common.conf.system", new="mock") def test_createrepo_filter_last_batch(self): module = self._create_module_with_filters(db_session, 3, koji.BUILD_STATES["COMPLETE"]) @@ -149,7 +149,7 @@ class TestMockModuleBuilder: rpm_names = [kobo.rpmlib.parse_nvr(rpm)["name"] for rpm in pkglist.split("\n")] assert "ed" not in rpm_names - @mock.patch("module_build_service.conf.system", new="mock") + @mock.patch("module_build_service.common.conf.system", new="mock") def test_createrepo_not_last_batch(self): module = self._create_module_with_filters(db_session, 2, koji.BUILD_STATES["COMPLETE"]) @@ -174,7 +174,7 @@ class TestMockModuleBuilder: rpm_names = [kobo.rpmlib.parse_nvr(rpm)["name"] for rpm in pkglist.split("\n")] assert "ed" in rpm_names - @mock.patch("module_build_service.conf.system", new="mock") + @mock.patch("module_build_service.common.conf.system", new="mock") def test_createrepo_empty_rmp_list(self): module = self._create_module_with_filters(db_session, 3, koji.BUILD_STATES["COMPLETE"]) @@ -194,7 +194,7 @@ class TestMockModuleBuilderAddRepos: def setup_method(self, test_method): clean_database(add_platform_module=False) - @mock.patch("module_build_service.conf.system", new="mock") + @mock.patch("module_build_service.common.conf.system", new="mock") @mock.patch( "module_build_service.common.config.Config.base_module_repofiles", new_callable=mock.PropertyMock, diff --git a/tests/test_common/test_config.py b/tests/test_common/test_config.py index 275d9a19..ce525521 100644 --- a/tests/test_common/test_config.py +++ b/tests/test_common/test_config.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import os.path -from module_build_service import conf +from module_build_service.common.config import conf class TestConfig: diff --git a/tests/test_common/test_logger.py b/tests/test_common/test_logger.py index d5ef2f5b..9c6156b4 100644 --- a/tests/test_common/test_logger.py +++ b/tests/test_common/test_logger.py @@ -6,8 +6,7 @@ from os import path import shutil import tempfile -from module_build_service import log -from module_build_service.common import models +from module_build_service.common import log, models from module_build_service.common.logger import ModuleBuildLogs from module_build_service.scheduler.consumer import MBSConsumer from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_common/test_models/test_models.py b/tests/test_common/test_models/test_models.py index 65b0463a..426172a6 100644 --- a/tests/test_common/test_models/test_models.py +++ b/tests/test_common/test_models/test_models.py @@ -5,7 +5,7 @@ from __future__ import absolute_import from mock import patch import pytest -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.common.models import ComponentBuild, ComponentBuildTrace, ModuleBuild from module_build_service.common.utils import load_mmd, mmd_to_str from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_common/test_monitor.py b/tests/test_common/test_monitor.py index d72f6fb4..88f3094f 100644 --- a/tests/test_common/test_monitor.py +++ b/tests/test_common/test_monitor.py @@ -8,8 +8,8 @@ import pytest import requests from six.moves import reload_module -from module_build_service import app, conf -from module_build_service.common import models +from module_build_service import app +from module_build_service.common import conf, models import module_build_service.common.monitor from module_build_service.scheduler.db_session import db_session from tests import clean_database, init_data, make_module_in_db diff --git a/tests/test_common/test_resolve.py b/tests/test_common/test_resolve.py index 0ca17f94..52e30608 100644 --- a/tests/test_common/test_resolve.py +++ b/tests/test_common/test_resolve.py @@ -5,8 +5,8 @@ from __future__ import absolute_import from mock import patch, PropertyMock import pytest -from module_build_service import Modulemd from module_build_service.common import models +from module_build_service.common.modulemd import Modulemd from module_build_service.common.utils import load_mmd from module_build_service.common.resolve import get_base_module_mmds from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_resolver/test_db.py b/tests/test_resolver/test_db.py index 26974511..4020879c 100644 --- a/tests/test_resolver/test_db.py +++ b/tests/test_resolver/test_db.py @@ -7,11 +7,12 @@ from datetime import datetime from mock import patch, PropertyMock import pytest -from module_build_service import conf, Modulemd from module_build_service.builder.MockModuleBuilder import load_local_builds from module_build_service.common import models +from module_build_service.common.config import conf from module_build_service.common.errors import UnprocessableEntity from module_build_service.common.models import ModuleBuild +from module_build_service.common.modulemd import Modulemd from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str import module_build_service.resolver as mbs_resolver from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_resolver/test_koji.py b/tests/test_resolver/test_koji.py index 415fc802..44c2961f 100644 --- a/tests/test_resolver/test_koji.py +++ b/tests/test_resolver/test_koji.py @@ -6,7 +6,7 @@ from datetime import datetime from mock import patch, MagicMock import pytest -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.common.models import ModuleBuild, BUILD_STATES from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str import module_build_service.resolver as mbs_resolver diff --git a/tests/test_resolver/test_local.py b/tests/test_resolver/test_local.py index c01a57cc..2e39fb80 100644 --- a/tests/test_resolver/test_local.py +++ b/tests/test_resolver/test_local.py @@ -5,7 +5,7 @@ from datetime import datetime import pytest -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.common.models import ModuleBuild from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_resolver/test_mbs.py b/tests/test_resolver/test_mbs.py index d5930fbf..368faa24 100644 --- a/tests/test_resolver/test_mbs.py +++ b/tests/test_resolver/test_mbs.py @@ -4,9 +4,10 @@ from __future__ import absolute_import from mock import patch, PropertyMock, Mock, call -from module_build_service import app, conf +from module_build_service import app from module_build_service.builder.MockModuleBuilder import load_local_builds import module_build_service.common.models +from module_build_service.common import conf from module_build_service.common.utils import load_mmd, mmd_to_str import module_build_service.resolver as mbs_resolver from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_scheduler/test_batches.py b/tests/test_scheduler/test_batches.py index 0d3beb9a..161fc5af 100644 --- a/tests/test_scheduler/test_batches.py +++ b/tests/test_scheduler/test_batches.py @@ -7,7 +7,7 @@ import mock from mock import patch import pytest -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.builder import GenericBuilder from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder from module_build_service.builder.utils import validate_koji_tag diff --git a/tests/test_scheduler/test_celery_route_task.py b/tests/test_scheduler/test_celery_route_task.py index c8793ee5..25c8d779 100644 --- a/tests/test_scheduler/test_celery_route_task.py +++ b/tests/test_scheduler/test_celery_route_task.py @@ -4,7 +4,8 @@ from __future__ import absolute_import import mock -from module_build_service import celery_app, conf +from module_build_service.common.config import conf +from module_build_service.scheduler import celery_app from module_build_service.scheduler.handlers import components, greenwave, modules, repos, tags from module_build_service.scheduler.producer import fail_lost_builds from tests import scheduler_init_data diff --git a/tests/test_scheduler/test_db_session.py b/tests/test_scheduler/test_db_session.py index ec32debc..da4386a0 100644 --- a/tests/test_scheduler/test_db_session.py +++ b/tests/test_scheduler/test_db_session.py @@ -3,8 +3,8 @@ from mock import patch -from module_build_service import conf from module_build_service.common import models +from module_build_service.common.config import conf from module_build_service.scheduler.db_session import db_session from tests import clean_database, make_module_in_db diff --git a/tests/test_scheduler/test_default_modules.py b/tests/test_scheduler/test_default_modules.py index 64186a4b..f7334db4 100644 --- a/tests/test_scheduler/test_default_modules.py +++ b/tests/test_scheduler/test_default_modules.py @@ -8,7 +8,7 @@ import dnf from mock import call, Mock, patch, PropertyMock import pytest -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.common.errors import UnprocessableEntity from module_build_service.common.models import ModuleBuild from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str diff --git a/tests/test_scheduler/test_greenwave_handler.py b/tests/test_scheduler/test_greenwave_handler.py index 6b27b122..076b8f4b 100644 --- a/tests/test_scheduler/test_greenwave_handler.py +++ b/tests/test_scheduler/test_greenwave_handler.py @@ -7,7 +7,7 @@ import pytest from sqlalchemy import func -from module_build_service import conf +from module_build_service.common.config import conf import module_build_service.common.config from module_build_service.common.models import BUILD_STATES, ModuleBuild from module_build_service.scheduler.consumer import MBSConsumer diff --git a/tests/test_scheduler/test_module_init.py b/tests/test_scheduler/test_module_init.py index 6abb23ba..3978c257 100644 --- a/tests/test_scheduler/test_module_init.py +++ b/tests/test_scheduler/test_module_init.py @@ -5,7 +5,7 @@ import os from mock import patch, PropertyMock -from module_build_service import build_logs, conf +from module_build_service.common import build_logs, conf from module_build_service.common.models import BUILD_STATES, ModuleBuild from module_build_service.common.utils import load_mmd, mmd_to_str from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index 9aca0f62..4a31d717 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -8,8 +8,9 @@ import mock from mock import patch import pytest -from module_build_service import build_logs, conf, Modulemd +from module_build_service.common import build_logs, conf from module_build_service.common.models import ComponentBuild, ModuleBuild +from module_build_service.common.modulemd import Modulemd import module_build_service.resolver from module_build_service.scheduler.db_session import db_session import module_build_service.scheduler.handlers.modules diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index 6fb28fbf..bbc836ef 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -9,7 +9,7 @@ import mock from mock import call, patch import pytest -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.common import models from module_build_service.scheduler import producer from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_scheduler/test_reuse.py b/tests/test_scheduler/test_reuse.py index 2cdef47b..a7e701af 100644 --- a/tests/test_scheduler/test_reuse.py +++ b/tests/test_scheduler/test_reuse.py @@ -6,8 +6,8 @@ import mock import pytest from sqlalchemy.orm.session import make_transient -from module_build_service import Modulemd from module_build_service.common import models +from module_build_service.common.modulemd import Modulemd from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str from module_build_service.scheduler.db_session import db_session from module_build_service.scheduler.reuse import get_reusable_component, get_reusable_module diff --git a/tests/test_scheduler/test_submit.py b/tests/test_scheduler/test_submit.py index 4bee0e22..c98b7f82 100644 --- a/tests/test_scheduler/test_submit.py +++ b/tests/test_scheduler/test_submit.py @@ -6,8 +6,8 @@ from datetime import datetime import mock import pytest -from module_build_service import app, conf -from module_build_service.common import models +from module_build_service import app +from module_build_service.common import conf, models from module_build_service.common.errors import UnprocessableEntity from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str from module_build_service.scheduler.db_session import db_session diff --git a/tests/test_scheduler/test_ursine.py b/tests/test_scheduler/test_ursine.py index 68176fc8..3a0df126 100644 --- a/tests/test_scheduler/test_ursine.py +++ b/tests/test_scheduler/test_ursine.py @@ -4,7 +4,7 @@ from __future__ import absolute_import from mock import patch, Mock -from module_build_service import conf +from module_build_service.common.config import conf from module_build_service.scheduler import ursine from tests import make_module, make_module_in_db, clean_database