From 6907e02c13e921bebee851df9303a1ce253d4eae Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Fri, 6 Dec 2019 11:34:06 +0800 Subject: [PATCH] Revert the config split change Revert the changes in a207d97 and 9a2efb3, at this moment, some code relies on both settings from frontend and backend, and it's inconvenient to run unittests with config split up to frontend only and backend only. --- conf/backend_config.py | 86 --------------- conf/config.py | 178 +++++++++++++++++++++++++++++++ conf/web_config.py | 70 ------------ module_build_service/__init__.py | 9 +- module_build_service/config.py | 57 +++------- module_build_service/models.py | 2 +- setup.py | 3 +- tests/test_monitor.py | 2 +- 8 files changed, 194 insertions(+), 213 deletions(-) delete mode 100644 conf/backend_config.py create mode 100644 conf/config.py delete mode 100644 conf/web_config.py diff --git a/conf/backend_config.py b/conf/backend_config.py deleted file mode 100644 index 43855d6a..00000000 --- a/conf/backend_config.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from os import environ, path - -confdir = path.abspath(path.dirname(__file__)) -dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir - - -class BackendConfiguration(object): - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - - # How often should we resort to polling, in seconds - # Set to zero to disable polling - POLLING_INTERVAL = 600 - - # Configs for running tasks asynchronously with Celery - # For details of Celery configs, refer to Celery documentation: - # https://docs.celeryproject.org/en/latest/userguide/configuration.html - # - # Each config name consists of namespace CELERY_ and the new Celery config - # name converted to upper case. For example the broker url, Celery config - # name is broker_url, then as you can see below, the corresponding config - # name in MBS is CELERY_BROKER_URL. - CELERY_BROKER_URL = "" - CELERY_RESULT_BACKEND = "" - CELERY_IMPORTS = [ - "module_build_service.scheduler.handlers.components", - "module_build_service.scheduler.handlers.modules", - "module_build_service.scheduler.handlers.repos", - "module_build_service.scheduler.handlers.tags", - "module_build_service.scheduler.handlers.greenwave", - ] - - -class TestConfiguration(BackendConfiguration): - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = environ.get( - "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) - DEBUG = True - MESSAGING = "in_memory" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - -class ProdConfiguration(BackendConfiguration): - pass - - -class LocalBuildConfiguration(BackendConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True - CELERY_BROKER_URL = "redis://localhost:6379/0" - CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/config.py b/conf/config.py new file mode 100644 index 00000000..270d994c --- /dev/null +++ b/conf/config.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from os import environ, path + +# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are +# declared properly somewhere/somehow +confdir = path.abspath(path.dirname(__file__)) +# use parent dir as dbdir else fallback to current dir +dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir + + +class BaseConfiguration(object): + DEBUG = False + # Make this random (used to generate session keys) + SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" + SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) + SQLALCHEMY_TRACK_MODIFICATIONS = True + # Where we should run when running "manage.py run" directly. + HOST = "0.0.0.0" + PORT = 5000 + + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + + SYSTEM = "koji" + MESSAGING = "fedmsg" # or amq + MESSAGING_TOPIC_PREFIX = ["org.fedoraproject.prod"] + KOJI_CONFIG = "/etc/module-build-service/koji.conf" + KOJI_PROFILE = "koji" + ARCHES = ["i686", "armv7hl", "x86_64"] + ALLOW_ARCH_OVERRIDE = False + KOJI_REPOSITORY_URL = "https://kojipkgs.fedoraproject.org/repos" + KOJI_TAG_PREFIXES = ["module", "scrmod"] + KOJI_ENABLE_CONTENT_GENERATOR = True + CHECK_FOR_EOL = False + PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" + PDC_INSECURE = False + PDC_DEVELOP = True + SCMURLS = ["https://src.fedoraproject.org/modules/"] + YAML_SUBMIT_ALLOWED = False + + # How often should we resort to polling, in seconds + # Set to zero to disable polling + POLLING_INTERVAL = 600 + + # Determines how many builds that can be submitted to the builder + # and be in the build state at a time. Set this to 0 for no restrictions + NUM_CONCURRENT_BUILDS = 5 + + ALLOW_CUSTOM_SCMURLS = False + + RPMS_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/rpms/" + RPMS_ALLOW_REPOSITORY = False + RPMS_DEFAULT_CACHE = "http://pkgs.fedoraproject.org/repo/pkgs/" + RPMS_ALLOW_CACHE = False + + MODULES_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/modules/" + MODULES_ALLOW_REPOSITORY = False + MODULES_ALLOW_SCRATCH = False + + ALLOWED_GROUPS = {"packager"} + + ALLOWED_GROUPS_TO_IMPORT_MODULE = set() + + # Available backends are: console and file + LOG_BACKEND = "console" + + # Path to log file when LOG_BACKEND is set to "file". + LOG_FILE = "module_build_service.log" + + # Available log levels are: debug, info, warn, error. + LOG_LEVEL = "info" + + # Settings for Kerberos + KRB_KEYTAB = None + KRB_PRINCIPAL = None + + # AMQ prefixed variables are required only while using 'amq' as messaging backend + # Addresses to listen to + AMQ_RECV_ADDRESSES = [ + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji", + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service", + ] + # Address for sending messages + AMQ_DEST_ADDRESS = \ + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service" + AMQ_CERT_FILE = "/etc/module_build_service/msg-m8y-client.crt" + AMQ_PRIVATE_KEY_FILE = "/etc/module_build_service/msg-m8y-client.key" + AMQ_TRUSTED_CERT_FILE = "/etc/module_build_service/Root-CA.crt" + + # Disable Client Authorization + NO_AUTH = False + + # Configs for running tasks asynchronously with Celery + # For details of Celery configs, refer to Celery documentation: + # https://docs.celeryproject.org/en/latest/userguide/configuration.html + # + # Each config name consists of namespace CELERY_ and the new Celery config + # name converted to upper case. For example the broker url, Celery config + # name is broker_url, then as you can below, the corresponding config name + # in MBS is CELERY_BROKER_URL. + CELERY_BROKER_URL = "" + CELERY_RESULT_BACKEND = "" + CELERY_IMPORTS = [ + "module_build_service.scheduler.handlers.components", + "module_build_service.scheduler.handlers.modules", + "module_build_service.scheduler.handlers.repos", + "module_build_service.scheduler.handlers.tags", + "module_build_service.scheduler.handlers.greenwave", + ] + + +class TestConfiguration(BaseConfiguration): + BUILD_LOGS_DIR = "/tmp" + BUILD_LOGS_NAME_FORMAT = "build-{id}.log" + LOG_BACKEND = "console" + LOG_LEVEL = "debug" + SQLALCHEMY_DATABASE_URI = environ.get( + "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) + DEBUG = True + MESSAGING = "in_memory" + PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + # SCM network-related values, in seconds + SCM_NET_TIMEOUT = 0.1 + SCM_NET_RETRY_INTERVAL = 0.1 + + KOJI_CONFIG = "./conf/koji.conf" + KOJI_PROFILE = "staging" + SERVER_NAME = "localhost" + + KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" + SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] + AUTH_METHOD = "oidc" + RESOLVER = "db" + + ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} + + # Greenwave configuration + GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" + GREENWAVE_DECISION_CONTEXT = "test_dec_context" + GREENWAVE_SUBJECT_TYPE = "some-module" + + STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} + + +class ProdConfiguration(BaseConfiguration): + pass + + +class LocalBuildConfiguration(BaseConfiguration): + CACHE_DIR = "~/modulebuild/cache" + LOG_LEVEL = "debug" + MESSAGING = "in_memory" + + ARCH_AUTODETECT = True + ARCH_FALLBACK = "x86_64" + + ALLOW_CUSTOM_SCMURLS = True + RESOLVER = "mbs" + RPMS_ALLOW_REPOSITORY = True + MODULES_ALLOW_REPOSITORY = True + + +class OfflineLocalBuildConfiguration(LocalBuildConfiguration): + RESOLVER = "local" + + +class DevConfiguration(LocalBuildConfiguration): + DEBUG = True + LOG_BACKEND = "console" + + CELERY_BROKER_URL = "redis://localhost:6379/0" + CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/web_config.py b/conf/web_config.py deleted file mode 100644 index 465d5a8d..00000000 --- a/conf/web_config.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from os import environ, path - -confdir = path.abspath(path.dirname(__file__)) -dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir - - -class WebConfiguration(object): - SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - - # Where we should run when running "manage.py run" directly. - HOST = "0.0.0.0" - PORT = 5000 - - -class TestConfiguration(WebConfiguration): - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = environ.get( - "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) - DEBUG = True - MESSAGING = "in_memory" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - - ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - -class ProdConfiguration(WebConfiguration): - pass - - -class LocalBuildConfiguration(WebConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - ALLOW_CUSTOM_SCMURLS = True - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index b2eae40f..236aa8b6 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -18,9 +18,7 @@ for a number of tasks: infrastructure services can pick up the work. """ -import os.path import pkg_resources -import sys from celery import Celery from flask import Flask, has_app_context, url_for from flask_sqlalchemy import SQLAlchemy @@ -48,12 +46,7 @@ api_version = 2 app = Flask(__name__) app.wsgi_app = ReverseProxy(app.wsgi_app) -backend_commands = ("fedmsg-hub", "celery", "build_module_locally") -if any([os.path.basename(arg).startswith(backend_commands) for arg in sys.argv]): - # running as backend - conf = init_config(app, backend=True) -else: - conf = init_config(app) +conf = init_config(app) celery_app = Celery("module-build-service") # Convert config names specific for Celery like this: diff --git a/module_build_service/config.py b/module_build_service/config.py index 2a7dc313..cb7582fd 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -24,19 +24,11 @@ SUPPORTED_RESOLVERS = { } -def init_config(app, backend=False): +def init_config(app): """ Configure MBS and the Flask app - By default, this create a config object with WebConfig, if backend is specified, - create a config object with BackendConfig. """ config_module = None - if backend: - config_module_name = "backend_config" - config_file = "/etc/module-build-service/backend_config.py" - else: - config_module_name = "web_config" - config_file = "/etc/module-build-service/web_config.py" - + config_file = "/etc/module-build-service/config.py" config_section = "DevConfiguration" # automagically detect production environment: @@ -74,14 +66,13 @@ def init_config(app, backend=False): if "MBS_CONFIG_SECTION" in app.request.environ: config_section = app.request.environ["MBS_CONFIG_SECTION"] - test_env = False - dev_env = False - true_options = ("1", "on", "true", "y", "yes") # TestConfiguration shall only be used for running tests, otherwise... if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]): - test_env = True config_section = "TestConfiguration" + from conf import config + + config_module = config # ...MODULE_BUILD_SERVICE_DEVELOPER_ENV has always the last word # and overrides anything previously set before! # Again, check Flask app (preferably) or fallback to os.environ. @@ -89,16 +80,15 @@ def init_config(app, backend=False): # -> /conf/config.py. elif flask_app_env and "MODULE_BUILD_SERVICE_DEVELOPER_ENV" in app.request.environ: if app.request.environ["MODULE_BUILD_SERVICE_DEVELOPER_ENV"].lower() in true_options: - dev_env = True config_section = "DevConfiguration" + from conf import config + + config_module = config elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options: - dev_env = True config_section = "DevConfiguration" + from conf import config - if test_env or dev_env: - import importlib - config_module = importlib.import_module("conf.%s" % config_module_name) - + config_module = config # try loading configuration from file if not config_module: try: @@ -108,10 +98,7 @@ def init_config(app, backend=False): # finally configure MBS and the Flask app config_section_obj = getattr(config_module, config_section) - if backend: - conf = BackendConfig(config_section_obj) - else: - conf = WebConfig(config_section_obj) + conf = Config(config_section_obj) app.config.from_object(config_section_obj) return conf @@ -125,7 +112,7 @@ class Path: class Config(object): - """Class representing the orchestrator common configuration.""" + """Class representing the orchestrator configuration.""" _defaults = { "debug": {"type": bool, "default": False, "desc": "Debug mode"}, @@ -934,23 +921,3 @@ class Config(object): if i < 1: raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1") self._num_threads_for_build_submissions = i - - -class WebConfig(Config): - """Class representing the orchestrator frontend web configuration.""" - _web_defaults = { - } - - def __init__(self, conf_section_obj): - self._defaults.update(self._web_defaults) - super(WebConfig, self).__init__(conf_section_obj) - - -class BackendConfig(Config): - """Class representing the orchestrator backend workers configuration.""" - _backend_defaults = { - } - - def __init__(self, conf_section_obj): - self._defaults.update(self._backend_defaults) - super(BackendConfig, self).__init__(conf_section_obj) diff --git a/module_build_service/models.py b/module_build_service/models.py index 0c602ac9..7ec35b6d 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -663,7 +663,7 @@ class ModuleBuild(MBSBase): bus. :param db_session: SQLAlchemy session object. - :param conf: MBS config object returned from function :func:`init_config`, + :param conf: MBS config object returned from function :func:`init_config` which contains loaded configs. :type conf: :class:`Config` :param int state: the state value to transition to. Refer to ``BUILD_STATES``. diff --git a/setup.py b/setup.py index db38696f..99c2f29c 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,7 @@ setup( "/etc/module-build-service/", [ "conf/cacert.pem", - "conf/web_config.py", - "conf/backend_config.py", + "conf/config.py", "conf/koji.conf", "conf/mock.cfg", "conf/yum.conf", diff --git a/tests/test_monitor.py b/tests/test_monitor.py index f45aa55c..ff9e3228 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -8,7 +8,7 @@ import module_build_service.config as mbs_config import module_build_service.monitor from module_build_service import models from module_build_service.db_session import db_session -from conf.web_config import TestConfiguration +from conf.config import TestConfiguration from six.moves import reload_module from tests import app, init_data, make_module_in_db