Create config for frontend or backend seperately

1. init_web_config: create Config object for frontend, load
   configuration from `web_config.py`.
2. init_backend_config: create Config for backend, load configuration
   from `backend_config.py`.

And two new classes inherit from `Config` in config.py:

1. WebConfig: representing the orchestrator frontend web configuration
2. BackendConfig: representing the orchestrator backend workers
                  configuration

Before calling init_{web,backend}_config, check sys.argv, if
"fedmsg-hub*", "celery" or "build_module_locally" is present, it's
running as backend.

conf/backend_config.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+from os import environ, path
+
+confdir = path.abspath(path.dirname(__file__))
+dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir
+
+
+class BackendConfiguration(object):
+    # How often should we resort to polling, in seconds
+    # Set to zero to disable polling
+    POLLING_INTERVAL = 600
+
+    # Configs for running tasks asynchronously with Celery
+    # For details of Celery configs, refer to Celery documentation:
+    # https://docs.celeryproject.org/en/latest/userguide/configuration.html
+    #
+    # Each config name consists of namespace CELERY_ and the new Celery config
+    # name converted to upper case. For example the broker url, Celery config
+    # name is broker_url, then as you can see below, the corresponding config
+    # name in MBS is CELERY_BROKER_URL.
+    CELERY_BROKER_URL = ""
+    CELERY_RESULT_BACKEND = ""
+    CELERY_IMPORTS = []
+
+
+class TestConfiguration(BackendConfiguration):
+    LOG_LEVEL = "debug"
+    SQLALCHEMY_DATABASE_URI = environ.get(
+        "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db")))
+    DEBUG = True
+    MESSAGING = "in_memory"
+
+    # Global network-related values, in seconds
+    NET_TIMEOUT = 3
+    NET_RETRY_INTERVAL = 1
+    # SCM network-related values, in seconds
+    SCM_NET_TIMEOUT = 0.1
+    SCM_NET_RETRY_INTERVAL = 0.1
+
+    KOJI_CONFIG = "./conf/koji.conf"
+    KOJI_PROFILE = "staging"
+    SERVER_NAME = "localhost"
+
+    KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos"
+    SCMURLS = ["https://src.stg.fedoraproject.org/modules/"]
+
+    # Greenwave configuration
+    GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
+    GREENWAVE_DECISION_CONTEXT = "test_dec_context"
+    GREENWAVE_SUBJECT_TYPE = "some-module"
+
+    STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
+
+
+class ProdConfiguration(BackendConfiguration):
+    pass
+
+
+class LocalBuildConfiguration(BackendConfiguration):
+    CACHE_DIR = "~/modulebuild/cache"
+    LOG_LEVEL = "debug"
+    MESSAGING = "in_memory"
+
+    RESOLVER = "mbs"
+    RPMS_ALLOW_REPOSITORY = True
+    MODULES_ALLOW_REPOSITORY = True
+
+
+class OfflineLocalBuildConfiguration(LocalBuildConfiguration):
+    RESOLVER = "local"
+
+
+class DevConfiguration(LocalBuildConfiguration):
+    DEBUG = True
+    CELERY_BROKER_URL = "redis://localhost:6379/0"
+    CELERY_RESULT_BACKEND = "redis://localhost:6379/0"

conf/config.py

@@ -1,172 +0,0 @@
-# -*- coding: utf-8 -*-
-# SPDX-License-Identifier: MIT
-from os import environ, path
-
-# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are
-# declared properly somewhere/somehow
-confdir = path.abspath(path.dirname(__file__))
-# use parent dir as dbdir else fallback to current dir
-dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir
-
-
-class BaseConfiguration(object):
-    DEBUG = False
-    # Make this random (used to generate session keys)
-    SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0"
-    SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db"))
-    SQLALCHEMY_TRACK_MODIFICATIONS = True
-    # Where we should run when running "manage.py run" directly.
-    HOST = "0.0.0.0"
-    PORT = 5000
-
-    # Global network-related values, in seconds
-    NET_TIMEOUT = 120
-    NET_RETRY_INTERVAL = 30
-
-    SYSTEM = "koji"
-    MESSAGING = "fedmsg"  # or amq
-    MESSAGING_TOPIC_PREFIX = ["org.fedoraproject.prod"]
-    KOJI_CONFIG = "/etc/module-build-service/koji.conf"
-    KOJI_PROFILE = "koji"
-    ARCHES = ["i686", "armv7hl", "x86_64"]
-    ALLOW_ARCH_OVERRIDE = False
-    KOJI_REPOSITORY_URL = "https://kojipkgs.fedoraproject.org/repos"
-    KOJI_TAG_PREFIXES = ["module", "scrmod"]
-    KOJI_ENABLE_CONTENT_GENERATOR = True
-    CHECK_FOR_EOL = False
-    PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1"
-    PDC_INSECURE = False
-    PDC_DEVELOP = True
-    SCMURLS = ["https://src.fedoraproject.org/modules/"]
-    YAML_SUBMIT_ALLOWED = False
-
-    # How often should we resort to polling, in seconds
-    # Set to zero to disable polling
-    POLLING_INTERVAL = 600
-
-    # Determines how many builds that can be submitted to the builder
-    # and be in the build state at a time. Set this to 0 for no restrictions
-    NUM_CONCURRENT_BUILDS = 5
-
-    ALLOW_CUSTOM_SCMURLS = False
-
-    RPMS_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/rpms/"
-    RPMS_ALLOW_REPOSITORY = False
-    RPMS_DEFAULT_CACHE = "http://pkgs.fedoraproject.org/repo/pkgs/"
-    RPMS_ALLOW_CACHE = False
-
-    MODULES_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/modules/"
-    MODULES_ALLOW_REPOSITORY = False
-    MODULES_ALLOW_SCRATCH = False
-
-    ALLOWED_GROUPS = {"packager"}
-
-    ALLOWED_GROUPS_TO_IMPORT_MODULE = set()
-
-    # Available backends are: console and file
-    LOG_BACKEND = "console"
-
-    # Path to log file when LOG_BACKEND is set to "file".
-    LOG_FILE = "module_build_service.log"
-
-    # Available log levels are: debug, info, warn, error.
-    LOG_LEVEL = "info"
-
-    # Settings for Kerberos
-    KRB_KEYTAB = None
-    KRB_PRINCIPAL = None
-
-    # AMQ prefixed variables are required only while using 'amq' as messaging backend
-    # Addresses to listen to
-    AMQ_RECV_ADDRESSES = [
-        "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji",
-        "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service",
-    ]
-    # Address for sending messages
-    AMQ_DEST_ADDRESS = \
-        "amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service"
-    AMQ_CERT_FILE = "/etc/module_build_service/msg-m8y-client.crt"
-    AMQ_PRIVATE_KEY_FILE = "/etc/module_build_service/msg-m8y-client.key"
-    AMQ_TRUSTED_CERT_FILE = "/etc/module_build_service/Root-CA.crt"
-
-    # Disable Client Authorization
-    NO_AUTH = False
-
-    # Configs for running tasks asynchronously with Celery
-    # For details of Celery configs, refer to Celery documentation:
-    # https://docs.celeryproject.org/en/latest/userguide/configuration.html
-    #
-    # Each config name consists of namespace CELERY_ and the new Celery config
-    # name converted to upper case. For example the broker url, Celery config
-    # name is broker_url, then as you can below, the corresponding config name
-    # in MBS is CELERY_BROKER_URL.
-    CELERY_BROKER_URL = ""
-    CELERY_RESULT_BACKEND = ""
-    CELERY_IMPORTS = []
-
-
-class TestConfiguration(BaseConfiguration):
-    BUILD_LOGS_DIR = "/tmp"
-    BUILD_LOGS_NAME_FORMAT = "build-{id}.log"
-    LOG_BACKEND = "console"
-    LOG_LEVEL = "debug"
-    SQLALCHEMY_DATABASE_URI = environ.get(
-        "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db")))
-    DEBUG = True
-    MESSAGING = "in_memory"
-    PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1"
-
-    # Global network-related values, in seconds
-    NET_TIMEOUT = 3
-    NET_RETRY_INTERVAL = 1
-    # SCM network-related values, in seconds
-    SCM_NET_TIMEOUT = 0.1
-    SCM_NET_RETRY_INTERVAL = 0.1
-
-    KOJI_CONFIG = "./conf/koji.conf"
-    KOJI_PROFILE = "staging"
-    SERVER_NAME = "localhost"
-
-    KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos"
-    SCMURLS = ["https://src.stg.fedoraproject.org/modules/"]
-    AUTH_METHOD = "oidc"
-    RESOLVER = "db"
-
-    ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"}
-
-    # Greenwave configuration
-    GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
-    GREENWAVE_DECISION_CONTEXT = "test_dec_context"
-    GREENWAVE_SUBJECT_TYPE = "some-module"
-
-    STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
-
-
-class ProdConfiguration(BaseConfiguration):
-    pass
-
-
-class LocalBuildConfiguration(BaseConfiguration):
-    CACHE_DIR = "~/modulebuild/cache"
-    LOG_LEVEL = "debug"
-    MESSAGING = "in_memory"
-
-    ARCH_AUTODETECT = True
-    ARCH_FALLBACK = "x86_64"
-
-    ALLOW_CUSTOM_SCMURLS = True
-    RESOLVER = "mbs"
-    RPMS_ALLOW_REPOSITORY = True
-    MODULES_ALLOW_REPOSITORY = True
-
-
-class OfflineLocalBuildConfiguration(LocalBuildConfiguration):
-    RESOLVER = "local"
-
-
-class DevConfiguration(LocalBuildConfiguration):
-    DEBUG = True
-    LOG_BACKEND = "console"
-
-    CELERY_BROKER_URL = "redis://localhost:6379/0"
-    CELERY_RESULT_BACKEND = "redis://localhost:6379/0"

conf/web_config.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+from os import environ, path
+
+confdir = path.abspath(path.dirname(__file__))
+dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir
+
+
+class WebConfiguration(object):
+    # Where we should run when running "manage.py run" directly.
+    HOST = "0.0.0.0"
+    PORT = 5000
+
+
+class TestConfiguration(WebConfiguration):
+    LOG_LEVEL = "debug"
+    SQLALCHEMY_DATABASE_URI = environ.get(
+        "DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db")))
+    DEBUG = True
+    MESSAGING = "in_memory"
+
+    # Global network-related values, in seconds
+    NET_TIMEOUT = 3
+    NET_RETRY_INTERVAL = 1
+    # SCM network-related values, in seconds
+    SCM_NET_TIMEOUT = 0.1
+    SCM_NET_RETRY_INTERVAL = 0.1
+
+    KOJI_CONFIG = "./conf/koji.conf"
+    KOJI_PROFILE = "staging"
+    SERVER_NAME = "localhost"
+
+    KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos"
+    SCMURLS = ["https://src.stg.fedoraproject.org/modules/"]
+
+    ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"}
+
+    # Greenwave configuration
+    GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
+    GREENWAVE_DECISION_CONTEXT = "test_dec_context"
+    GREENWAVE_SUBJECT_TYPE = "some-module"
+
+    STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
+
+
+class ProdConfiguration(WebConfiguration):
+    pass
+
+
+class LocalBuildConfiguration(WebConfiguration):
+    CACHE_DIR = "~/modulebuild/cache"
+    LOG_LEVEL = "debug"
+    MESSAGING = "in_memory"
+
+    ALLOW_CUSTOM_SCMURLS = True
+    RESOLVER = "mbs"
+    RPMS_ALLOW_REPOSITORY = True
+    MODULES_ALLOW_REPOSITORY = True
+
+
+class OfflineLocalBuildConfiguration(LocalBuildConfiguration):
+    RESOLVER = "local"
+
+
+class DevConfiguration(LocalBuildConfiguration):
+    DEBUG = True

module_build_service/__init__.py

@@ -18,7 +18,9 @@ for a number of tasks:
   infrastructure services can pick up the work.
 """
 
+import os.path
 import pkg_resources
+import sys
 from celery import Celery
 from flask import Flask, has_app_context, url_for
 from flask_sqlalchemy import SQLAlchemy
@@ -34,7 +36,7 @@ from module_build_service.logger import init_logging, ModuleBuildLogs, level_fla
 from module_build_service.errors import (
     ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound,
     Forbidden, json_error)
-from module_build_service.config import init_config
+from module_build_service.config import init_web_config, init_backend_config
 from module_build_service.proxy import ReverseProxy
 
 try:
@@ -46,7 +48,12 @@ api_version = 2
 app = Flask(__name__)
 app.wsgi_app = ReverseProxy(app.wsgi_app)
 
-conf = init_config(app)
+backend_commands = ("fedmsg-hub", "celery", "build_module_locally")
+if any([os.path.basename(arg).startswith(backend_commands) for arg in sys.argv]):
+    # running as backend
+    conf = init_backend_config()
+else:
+    conf = init_web_config(app)
 
 celery_app = Celery("module-build-service")
 # Convert config names specific for Celery like this:

module_build_service/builder/KojiModuleBuilder.py

@@ -474,7 +474,8 @@ class KojiModuleBuilder(GenericBuilder):
     def get_session(config, login=True):
         """Create and return a koji.ClientSession object
 
-        :param config: the config object returned from :meth:`init_config`.
+        :param config: the config object returned from :meth:`init_web_config` or
+            :meth:`init_backend_config`.
         :type config: :class:`Config`
         :param bool login: whether to log into the session. To login if True
             is passed, otherwise not to log into session.

module_build_service/config.py

@@ -24,11 +24,11 @@ SUPPORTED_RESOLVERS = {
 }
 
 
-def init_config(app):
+def init_web_config(app):
     """ Configure MBS and the Flask app
     """
     config_module = None
-    config_file = "/etc/module-build-service/config.py"
+    config_file = "/etc/module-build-service/web_config.py"
     config_section = "DevConfiguration"
 
     # automagically detect production environment:
@@ -70,9 +70,9 @@ def init_config(app):
     # TestConfiguration shall only be used for running tests, otherwise...
     if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]):
         config_section = "TestConfiguration"
-        from conf import config
+        from conf import web_config
 
-        config_module = config
+        config_module = web_config
     # ...MODULE_BUILD_SERVICE_DEVELOPER_ENV has always the last word
     # and overrides anything previously set before!
     # Again, check Flask app (preferably) or fallback to os.environ.
@@ -81,14 +81,14 @@ def init_config(app):
     elif flask_app_env and "MODULE_BUILD_SERVICE_DEVELOPER_ENV" in app.request.environ:
         if app.request.environ["MODULE_BUILD_SERVICE_DEVELOPER_ENV"].lower() in true_options:
             config_section = "DevConfiguration"
-            from conf import config
+            from conf import web_config
 
-            config_module = config
+            config_module = web_config
     elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options:
         config_section = "DevConfiguration"
-        from conf import config
+        from conf import web_config
 
-        config_module = config
+        config_module = web_config
     # try loading configuration from file
     if not config_module:
         try:
@@ -98,11 +98,63 @@ def init_config(app):
 
     # finally configure MBS and the Flask app
     config_section_obj = getattr(config_module, config_section)
-    conf = Config(config_section_obj)
+    conf = WebConfig(config_section_obj)
     app.config.from_object(config_section_obj)
     return conf
 
 
+def init_backend_config():
+    """ Configure MBS and backend workers
+    """
+    config_module = None
+    config_file = "/etc/module-build-service/backend_config.py"
+    config_section = "DevConfiguration"
+
+    try:
+        with open(config_file):
+            config_section = "ProdConfiguration"
+    except Exception:
+        pass
+
+    # Load LocalBuildConfiguration section in case we are building modules
+    # locally.
+    if "build_module_locally" in sys.argv:
+        if "--offline" in sys.argv:
+            config_section = "OfflineLocalBuildConfiguration"
+        else:
+            config_section = "LocalBuildConfiguration"
+
+    # try getting config_file from os.environ
+    if "MBS_CONFIG_FILE" in os.environ:
+        config_file = os.environ["MBS_CONFIG_FILE"]
+    # try getting config_section from os.environ
+    if "MBS_CONFIG_SECTION" in os.environ:
+        config_section = os.environ["MBS_CONFIG_SECTION"]
+
+    true_options = ("1", "on", "true", "y", "yes")
+    # TestConfiguration shall only be used for running tests, otherwise...
+    if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]):
+        config_section = "TestConfiguration"
+        from conf import backend_config
+
+        config_module = backend_config
+    elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options:
+        config_section = "DevConfiguration"
+        from conf import backend_config
+
+        config_module = backend_config
+    # try loading configuration from file
+    if not config_module:
+        try:
+            config_module = imp.load_source("mbs_runtime_config", config_file)
+        except Exception:
+            raise SystemError("Configuration file {} was not found.".format(config_file))
+
+    config_section_obj = getattr(config_module, config_section)
+    conf = BackendConfig(config_section_obj)
+    return conf
+
+
 class Path:
     """
     Config type for paths. Expands the users home directory.
@@ -112,7 +164,7 @@ class Path:
 
 
 class Config(object):
-    """Class representing the orchestrator configuration."""
+    """Class representing the orchestrator common configuration."""
 
     _defaults = {
         "debug": {"type": bool, "default": False, "desc": "Debug mode"},
@@ -244,7 +296,7 @@ class Config(object):
         "log_level": {"type": str, "default": 0, "desc": "Log level"},
         "build_logs_dir": {
             "type": Path,
-            "default": "",
+            "default": tempfile.gettempdir(),
             "desc": "Directory to store module build logs to.",
         },
         "build_logs_name_format": {
@@ -751,7 +803,7 @@ class Config(object):
 
     def _setifok_log_backend(self, s):
         if s is None:
-            self._log_backend = "console"
+            s = "console"
         elif s not in logger.supported_log_backends():
             raise ValueError("Unsupported log backend")
         self._log_backend = str(s)
@@ -908,3 +960,23 @@ class Config(object):
         if i < 1:
             raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1")
         self._num_threads_for_build_submissions = i
+
+
+class WebConfig(Config):
+    """Class representing the orchestrator frontend web configuration."""
+    _web_defaults = {
+    }
+
+    def __init__(self, conf_section_obj):
+        self._defaults.update(self._web_defaults)
+        super(WebConfig, self).__init__(conf_section_obj)
+
+
+class BackendConfig(Config):
+    """Class representing the orchestrator backend workers configuration."""
+    _backend_defaults = {
+    }
+
+    def __init__(self, conf_section_obj):
+        self._defaults.update(self._backend_defaults)
+        super(BackendConfig, self).__init__(conf_section_obj)

module_build_service/models.py

@@ -663,8 +663,8 @@ class ModuleBuild(MBSBase):
         bus.
 
         :param db_session: SQLAlchemy session object.
-        :param conf: MBS config object returned from function :func:`init_config`
-            which contains loaded configs.
+        :param conf: MBS config object returned from function :func:`init_web_config` or
+            :func:`init_backend_config`, which contains loaded configs.
         :type conf: :class:`Config`
         :param int state: the state value to transition to. Refer to ``BUILD_STATES``.
         :param str state_reason: optional reason of why to transform to ``state``.

setup.py

@@ -67,7 +67,8 @@ setup(
             "/etc/module-build-service/",
             [
                 "conf/cacert.pem",
-                "conf/config.py",
+                "conf/web_config.py",
+                "conf/backend_config.py",
                 "conf/koji.conf",
                 "conf/mock.cfg",
                 "conf/yum.conf",

tests/__init__.py

@@ -14,7 +14,7 @@ import koji
 import module_build_service
 from module_build_service import db
 from module_build_service.utils import get_rpm_release, import_mmd, mmd_to_str
-from module_build_service.config import init_config
+from module_build_service.config import init_web_config
 from module_build_service.models import (
     ModuleBuild, ModuleArch, ComponentBuild, VirtualStream,
     BUILD_STATES,
@@ -25,7 +25,7 @@ from module_build_service.db_session import db_session
 
 base_dir = os.path.dirname(__file__)
 app = module_build_service.app
-conf = init_config(app)
+conf = init_web_config(app)
 
 
 def staged_data_filename(filename):

tests/test_monitor.py

@@ -8,7 +8,7 @@ import module_build_service.config as mbs_config
 import module_build_service.monitor
 from module_build_service import models
 from module_build_service.db_session import db_session
-from conf.config import TestConfiguration
+from conf.web_config import TestConfiguration
 
 from six.moves import reload_module
 from tests import app, init_data, make_module_in_db