diff --git a/conf/backend_config.py b/conf/backend_config.py deleted file mode 100644 index 43855d6a..00000000 --- a/conf/backend_config.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from os import environ, path - -confdir = path.abspath(path.dirname(__file__)) -dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir - - -class BackendConfiguration(object): - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - - # How often should we resort to polling, in seconds - # Set to zero to disable polling - POLLING_INTERVAL = 600 - - # Configs for running tasks asynchronously with Celery - # For details of Celery configs, refer to Celery documentation: - # https://docs.celeryproject.org/en/latest/userguide/configuration.html - # - # Each config name consists of namespace CELERY_ and the new Celery config - # name converted to upper case. For example the broker url, Celery config - # name is broker_url, then as you can see below, the corresponding config - # name in MBS is CELERY_BROKER_URL. - CELERY_BROKER_URL = "" - CELERY_RESULT_BACKEND = "" - CELERY_IMPORTS = [ - "module_build_service.scheduler.handlers.components", - "module_build_service.scheduler.handlers.modules", - "module_build_service.scheduler.handlers.repos", - "module_build_service.scheduler.handlers.tags", - "module_build_service.scheduler.handlers.greenwave", - ] - - -class TestConfiguration(BackendConfiguration): - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = environ.get( - "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) - DEBUG = True - MESSAGING = "in_memory" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - -class ProdConfiguration(BackendConfiguration): - pass - - -class LocalBuildConfiguration(BackendConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True - CELERY_BROKER_URL = "redis://localhost:6379/0" - CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/config.py b/conf/config.py new file mode 100644 index 00000000..270d994c --- /dev/null +++ b/conf/config.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from os import environ, path + +# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are +# declared properly somewhere/somehow +confdir = path.abspath(path.dirname(__file__)) +# use parent dir as dbdir else fallback to current dir +dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir + + +class BaseConfiguration(object): + DEBUG = False + # Make this random (used to generate session keys) + SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" + SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) + SQLALCHEMY_TRACK_MODIFICATIONS = True + # Where we should run when running "manage.py run" directly. + HOST = "0.0.0.0" + PORT = 5000 + + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + + SYSTEM = "koji" + MESSAGING = "fedmsg" # or amq + MESSAGING_TOPIC_PREFIX = ["org.fedoraproject.prod"] + KOJI_CONFIG = "/etc/module-build-service/koji.conf" + KOJI_PROFILE = "koji" + ARCHES = ["i686", "armv7hl", "x86_64"] + ALLOW_ARCH_OVERRIDE = False + KOJI_REPOSITORY_URL = "https://kojipkgs.fedoraproject.org/repos" + KOJI_TAG_PREFIXES = ["module", "scrmod"] + KOJI_ENABLE_CONTENT_GENERATOR = True + CHECK_FOR_EOL = False + PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" + PDC_INSECURE = False + PDC_DEVELOP = True + SCMURLS = ["https://src.fedoraproject.org/modules/"] + YAML_SUBMIT_ALLOWED = False + + # How often should we resort to polling, in seconds + # Set to zero to disable polling + POLLING_INTERVAL = 600 + + # Determines how many builds that can be submitted to the builder + # and be in the build state at a time. Set this to 0 for no restrictions + NUM_CONCURRENT_BUILDS = 5 + + ALLOW_CUSTOM_SCMURLS = False + + RPMS_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/rpms/" + RPMS_ALLOW_REPOSITORY = False + RPMS_DEFAULT_CACHE = "http://pkgs.fedoraproject.org/repo/pkgs/" + RPMS_ALLOW_CACHE = False + + MODULES_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/modules/" + MODULES_ALLOW_REPOSITORY = False + MODULES_ALLOW_SCRATCH = False + + ALLOWED_GROUPS = {"packager"} + + ALLOWED_GROUPS_TO_IMPORT_MODULE = set() + + # Available backends are: console and file + LOG_BACKEND = "console" + + # Path to log file when LOG_BACKEND is set to "file". + LOG_FILE = "module_build_service.log" + + # Available log levels are: debug, info, warn, error. + LOG_LEVEL = "info" + + # Settings for Kerberos + KRB_KEYTAB = None + KRB_PRINCIPAL = None + + # AMQ prefixed variables are required only while using 'amq' as messaging backend + # Addresses to listen to + AMQ_RECV_ADDRESSES = [ + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji", + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service", + ] + # Address for sending messages + AMQ_DEST_ADDRESS = \ + "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service" + AMQ_CERT_FILE = "/etc/module_build_service/msg-m8y-client.crt" + AMQ_PRIVATE_KEY_FILE = "/etc/module_build_service/msg-m8y-client.key" + AMQ_TRUSTED_CERT_FILE = "/etc/module_build_service/Root-CA.crt" + + # Disable Client Authorization + NO_AUTH = False + + # Configs for running tasks asynchronously with Celery + # For details of Celery configs, refer to Celery documentation: + # https://docs.celeryproject.org/en/latest/userguide/configuration.html + # + # Each config name consists of namespace CELERY_ and the new Celery config + # name converted to upper case. For example the broker url, Celery config + # name is broker_url, then as you can below, the corresponding config name + # in MBS is CELERY_BROKER_URL. + CELERY_BROKER_URL = "" + CELERY_RESULT_BACKEND = "" + CELERY_IMPORTS = [ + "module_build_service.scheduler.handlers.components", + "module_build_service.scheduler.handlers.modules", + "module_build_service.scheduler.handlers.repos", + "module_build_service.scheduler.handlers.tags", + "module_build_service.scheduler.handlers.greenwave", + ] + + +class TestConfiguration(BaseConfiguration): + BUILD_LOGS_DIR = "/tmp" + BUILD_LOGS_NAME_FORMAT = "build-{id}.log" + LOG_BACKEND = "console" + LOG_LEVEL = "debug" + SQLALCHEMY_DATABASE_URI = environ.get( + "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) + DEBUG = True + MESSAGING = "in_memory" + PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1" + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + # SCM network-related values, in seconds + SCM_NET_TIMEOUT = 0.1 + SCM_NET_RETRY_INTERVAL = 0.1 + + KOJI_CONFIG = "./conf/koji.conf" + KOJI_PROFILE = "staging" + SERVER_NAME = "localhost" + + KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" + SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] + AUTH_METHOD = "oidc" + RESOLVER = "db" + + ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} + + # Greenwave configuration + GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" + GREENWAVE_DECISION_CONTEXT = "test_dec_context" + GREENWAVE_SUBJECT_TYPE = "some-module" + + STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} + + +class ProdConfiguration(BaseConfiguration): + pass + + +class LocalBuildConfiguration(BaseConfiguration): + CACHE_DIR = "~/modulebuild/cache" + LOG_LEVEL = "debug" + MESSAGING = "in_memory" + + ARCH_AUTODETECT = True + ARCH_FALLBACK = "x86_64" + + ALLOW_CUSTOM_SCMURLS = True + RESOLVER = "mbs" + RPMS_ALLOW_REPOSITORY = True + MODULES_ALLOW_REPOSITORY = True + + +class OfflineLocalBuildConfiguration(LocalBuildConfiguration): + RESOLVER = "local" + + +class DevConfiguration(LocalBuildConfiguration): + DEBUG = True + LOG_BACKEND = "console" + + CELERY_BROKER_URL = "redis://localhost:6379/0" + CELERY_RESULT_BACKEND = "redis://localhost:6379/0" diff --git a/conf/web_config.py b/conf/web_config.py deleted file mode 100644 index 465d5a8d..00000000 --- a/conf/web_config.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from os import environ, path - -confdir = path.abspath(path.dirname(__file__)) -dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir - - -class WebConfiguration(object): - SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0" - SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db")) - SQLALCHEMY_TRACK_MODIFICATIONS = True - - # Where we should run when running "manage.py run" directly. - HOST = "0.0.0.0" - PORT = 5000 - - -class TestConfiguration(WebConfiguration): - LOG_LEVEL = "debug" - SQLALCHEMY_DATABASE_URI = environ.get( - "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db"))) - DEBUG = True - MESSAGING = "in_memory" - - # Global network-related values, in seconds - NET_TIMEOUT = 3 - NET_RETRY_INTERVAL = 1 - # SCM network-related values, in seconds - SCM_NET_TIMEOUT = 0.1 - SCM_NET_RETRY_INTERVAL = 0.1 - - KOJI_CONFIG = "./conf/koji.conf" - KOJI_PROFILE = "staging" - SERVER_NAME = "localhost" - - KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" - SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] - - ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} - - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} - - -class ProdConfiguration(WebConfiguration): - pass - - -class LocalBuildConfiguration(WebConfiguration): - CACHE_DIR = "~/modulebuild/cache" - LOG_LEVEL = "debug" - MESSAGING = "in_memory" - - ALLOW_CUSTOM_SCMURLS = True - RESOLVER = "mbs" - RPMS_ALLOW_REPOSITORY = True - MODULES_ALLOW_REPOSITORY = True - - -class OfflineLocalBuildConfiguration(LocalBuildConfiguration): - RESOLVER = "local" - - -class DevConfiguration(LocalBuildConfiguration): - DEBUG = True diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index b2eae40f..236aa8b6 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -18,9 +18,7 @@ for a number of tasks: infrastructure services can pick up the work. """ -import os.path import pkg_resources -import sys from celery import Celery from flask import Flask, has_app_context, url_for from flask_sqlalchemy import SQLAlchemy @@ -48,12 +46,7 @@ api_version = 2 app = Flask(__name__) app.wsgi_app = ReverseProxy(app.wsgi_app) -backend_commands = ("fedmsg-hub", "celery", "build_module_locally") -if any([os.path.basename(arg).startswith(backend_commands) for arg in sys.argv]): - # running as backend - conf = init_config(app, backend=True) -else: - conf = init_config(app) +conf = init_config(app) celery_app = Celery("module-build-service") # Convert config names specific for Celery like this: diff --git a/module_build_service/config.py b/module_build_service/config.py index 2a7dc313..cb7582fd 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -24,19 +24,11 @@ SUPPORTED_RESOLVERS = { } -def init_config(app, backend=False): +def init_config(app): """ Configure MBS and the Flask app - By default, this create a config object with WebConfig, if backend is specified, - create a config object with BackendConfig. """ config_module = None - if backend: - config_module_name = "backend_config" - config_file = "/etc/module-build-service/backend_config.py" - else: - config_module_name = "web_config" - config_file = "/etc/module-build-service/web_config.py" - + config_file = "/etc/module-build-service/config.py" config_section = "DevConfiguration" # automagically detect production environment: @@ -74,14 +66,13 @@ def init_config(app, backend=False): if "MBS_CONFIG_SECTION" in app.request.environ: config_section = app.request.environ["MBS_CONFIG_SECTION"] - test_env = False - dev_env = False - true_options = ("1", "on", "true", "y", "yes") # TestConfiguration shall only be used for running tests, otherwise... if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]): - test_env = True config_section = "TestConfiguration" + from conf import config + + config_module = config # ...MODULE_BUILD_SERVICE_DEVELOPER_ENV has always the last word # and overrides anything previously set before! # Again, check Flask app (preferably) or fallback to os.environ. @@ -89,16 +80,15 @@ def init_config(app, backend=False): # -> /conf/config.py. elif flask_app_env and "MODULE_BUILD_SERVICE_DEVELOPER_ENV" in app.request.environ: if app.request.environ["MODULE_BUILD_SERVICE_DEVELOPER_ENV"].lower() in true_options: - dev_env = True config_section = "DevConfiguration" + from conf import config + + config_module = config elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options: - dev_env = True config_section = "DevConfiguration" + from conf import config - if test_env or dev_env: - import importlib - config_module = importlib.import_module("conf.%s" % config_module_name) - + config_module = config # try loading configuration from file if not config_module: try: @@ -108,10 +98,7 @@ def init_config(app, backend=False): # finally configure MBS and the Flask app config_section_obj = getattr(config_module, config_section) - if backend: - conf = BackendConfig(config_section_obj) - else: - conf = WebConfig(config_section_obj) + conf = Config(config_section_obj) app.config.from_object(config_section_obj) return conf @@ -125,7 +112,7 @@ class Path: class Config(object): - """Class representing the orchestrator common configuration.""" + """Class representing the orchestrator configuration.""" _defaults = { "debug": {"type": bool, "default": False, "desc": "Debug mode"}, @@ -934,23 +921,3 @@ class Config(object): if i < 1: raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1") self._num_threads_for_build_submissions = i - - -class WebConfig(Config): - """Class representing the orchestrator frontend web configuration.""" - _web_defaults = { - } - - def __init__(self, conf_section_obj): - self._defaults.update(self._web_defaults) - super(WebConfig, self).__init__(conf_section_obj) - - -class BackendConfig(Config): - """Class representing the orchestrator backend workers configuration.""" - _backend_defaults = { - } - - def __init__(self, conf_section_obj): - self._defaults.update(self._backend_defaults) - super(BackendConfig, self).__init__(conf_section_obj) diff --git a/module_build_service/models.py b/module_build_service/models.py index 0c602ac9..7ec35b6d 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -663,7 +663,7 @@ class ModuleBuild(MBSBase): bus. :param db_session: SQLAlchemy session object. - :param conf: MBS config object returned from function :func:`init_config`, + :param conf: MBS config object returned from function :func:`init_config` which contains loaded configs. :type conf: :class:`Config` :param int state: the state value to transition to. Refer to ``BUILD_STATES``. diff --git a/setup.py b/setup.py index db38696f..99c2f29c 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,7 @@ setup( "/etc/module-build-service/", [ "conf/cacert.pem", - "conf/web_config.py", - "conf/backend_config.py", + "conf/config.py", "conf/koji.conf", "conf/mock.cfg", "conf/yum.conf", diff --git a/tests/test_monitor.py b/tests/test_monitor.py index f45aa55c..ff9e3228 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -8,7 +8,7 @@ import module_build_service.config as mbs_config import module_build_service.monitor from module_build_service import models from module_build_service.db_session import db_session -from conf.web_config import TestConfiguration +from conf.config import TestConfiguration from six.moves import reload_module from tests import app, init_data, make_module_in_db