modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-01 09:50:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Revert the config split change

Browse Source 
Revert the changes in a207d97 and 9a2efb3, at this moment, some code
relies on both settings from frontend and backend, and it's inconvenient
to run unittests with config split up to frontend only and backend
only.
This commit is contained in:
Qixiang Wan
2019-12-06 11:34:06 +08:00
committed by mprahl
parent e40fea9b1b
commit 6907e02c13
8 changed files with 194 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
conf/backend_config.py
View File

@@ -1,86 +0,0 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from os import environ, path
confdir = path.abspath(path.dirname(__file__))
dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir
class BackendConfiguration(object):
SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db"))
SQLALCHEMY_TRACK_MODIFICATIONS = True
# How often should we resort to polling, in seconds
# Set to zero to disable polling
POLLING_INTERVAL = 600
# Configs for running tasks asynchronously with Celery
# For details of Celery configs, refer to Celery documentation:
# https://docs.celeryproject.org/en/latest/userguide/configuration.html
#
# Each config name consists of namespace CELERY_ and the new Celery config
# name converted to upper case. For example the broker url, Celery config
# name is broker_url, then as you can see below, the corresponding config
# name in MBS is CELERY_BROKER_URL.
CELERY_BROKER_URL = ""
CELERY_RESULT_BACKEND = ""
CELERY_IMPORTS = [
"module_build_service.scheduler.handlers.components",
"module_build_service.scheduler.handlers.modules",
"module_build_service.scheduler.handlers.repos",
"module_build_service.scheduler.handlers.tags",
"module_build_service.scheduler.handlers.greenwave",
]
class TestConfiguration(BackendConfiguration):
LOG_LEVEL = "debug"
SQLALCHEMY_DATABASE_URI = environ.get(
"DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db")))
DEBUG = True
MESSAGING = "in_memory"
# Global network-related values, in seconds
NET_TIMEOUT = 3
NET_RETRY_INTERVAL = 1
# SCM network-related values, in seconds
SCM_NET_TIMEOUT = 0.1
SCM_NET_RETRY_INTERVAL = 0.1
KOJI_CONFIG = "./conf/koji.conf"
KOJI_PROFILE = "staging"
SERVER_NAME = "localhost"
KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos"
SCMURLS = ["https://src.stg.fedoraproject.org/modules/"]
# Greenwave configuration
GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
GREENWAVE_DECISION_CONTEXT = "test_dec_context"
GREENWAVE_SUBJECT_TYPE = "some-module"
STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
class ProdConfiguration(BackendConfiguration):
pass
class LocalBuildConfiguration(BackendConfiguration):
CACHE_DIR = "~/modulebuild/cache"
LOG_LEVEL = "debug"
MESSAGING = "in_memory"
RESOLVER = "mbs"
RPMS_ALLOW_REPOSITORY = True
MODULES_ALLOW_REPOSITORY = True
class OfflineLocalBuildConfiguration(LocalBuildConfiguration):
RESOLVER = "local"
class DevConfiguration(LocalBuildConfiguration):
DEBUG = True
CELERY_BROKER_URL = "redis://localhost:6379/0"
CELERY_RESULT_BACKEND = "redis://localhost:6379/0"

178
conf/config.py Normal file
View File

@@ -0,0 +1,178 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from os import environ, path
# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are
# declared properly somewhere/somehow
confdir = path.abspath(path.dirname(__file__))
# use parent dir as dbdir else fallback to current dir
dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir
class BaseConfiguration(object):
DEBUG = False
# Make this random (used to generate session keys)
SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0"
SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db"))
SQLALCHEMY_TRACK_MODIFICATIONS = True
# Where we should run when running "manage.py run" directly.
HOST = "0.0.0.0"
PORT = 5000
# Global network-related values, in seconds
NET_TIMEOUT = 120
NET_RETRY_INTERVAL = 30
SYSTEM = "koji"
MESSAGING = "fedmsg" # or amq
MESSAGING_TOPIC_PREFIX = ["org.fedoraproject.prod"]
KOJI_CONFIG = "/etc/module-build-service/koji.conf"
KOJI_PROFILE = "koji"
ARCHES = ["i686", "armv7hl", "x86_64"]
ALLOW_ARCH_OVERRIDE = False
KOJI_REPOSITORY_URL = "https://kojipkgs.fedoraproject.org/repos"
KOJI_TAG_PREFIXES = ["module", "scrmod"]
KOJI_ENABLE_CONTENT_GENERATOR = True
CHECK_FOR_EOL = False
PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1"
PDC_INSECURE = False
PDC_DEVELOP = True
SCMURLS = ["https://src.fedoraproject.org/modules/"]
YAML_SUBMIT_ALLOWED = False
# How often should we resort to polling, in seconds
# Set to zero to disable polling
POLLING_INTERVAL = 600
# Determines how many builds that can be submitted to the builder
# and be in the build state at a time. Set this to 0 for no restrictions
NUM_CONCURRENT_BUILDS = 5
ALLOW_CUSTOM_SCMURLS = False
RPMS_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/rpms/"
RPMS_ALLOW_REPOSITORY = False
RPMS_DEFAULT_CACHE = "http://pkgs.fedoraproject.org/repo/pkgs/"
RPMS_ALLOW_CACHE = False
MODULES_DEFAULT_REPOSITORY = "https://src.fedoraproject.org/modules/"
MODULES_ALLOW_REPOSITORY = False
MODULES_ALLOW_SCRATCH = False
ALLOWED_GROUPS = {"packager"}
ALLOWED_GROUPS_TO_IMPORT_MODULE = set()
# Available backends are: console and file
LOG_BACKEND = "console"
# Path to log file when LOG_BACKEND is set to "file".
LOG_FILE = "module_build_service.log"
# Available log levels are: debug, info, warn, error.
LOG_LEVEL = "info"
# Settings for Kerberos
KRB_KEYTAB = None
KRB_PRINCIPAL = None
# AMQ prefixed variables are required only while using 'amq' as messaging backend
# Addresses to listen to
AMQ_RECV_ADDRESSES = [
"amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji",
"amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service",
]
# Address for sending messages
AMQ_DEST_ADDRESS = \
"amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service"
AMQ_CERT_FILE = "/etc/module_build_service/msg-m8y-client.crt"
AMQ_PRIVATE_KEY_FILE = "/etc/module_build_service/msg-m8y-client.key"
AMQ_TRUSTED_CERT_FILE = "/etc/module_build_service/Root-CA.crt"
# Disable Client Authorization
NO_AUTH = False
# Configs for running tasks asynchronously with Celery
# For details of Celery configs, refer to Celery documentation:
# https://docs.celeryproject.org/en/latest/userguide/configuration.html
#
# Each config name consists of namespace CELERY_ and the new Celery config
# name converted to upper case. For example the broker url, Celery config
# name is broker_url, then as you can below, the corresponding config name
# in MBS is CELERY_BROKER_URL.
CELERY_BROKER_URL = ""
CELERY_RESULT_BACKEND = ""
CELERY_IMPORTS = [
"module_build_service.scheduler.handlers.components",
"module_build_service.scheduler.handlers.modules",
"module_build_service.scheduler.handlers.repos",
"module_build_service.scheduler.handlers.tags",
"module_build_service.scheduler.handlers.greenwave",
]
class TestConfiguration(BaseConfiguration):
BUILD_LOGS_DIR = "/tmp"
BUILD_LOGS_NAME_FORMAT = "build-{id}.log"
LOG_BACKEND = "console"
LOG_LEVEL = "debug"
SQLALCHEMY_DATABASE_URI = environ.get(
"DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db")))
DEBUG = True
MESSAGING = "in_memory"
PDC_URL = "https://pdc.fedoraproject.org/rest_api/v1"
# Global network-related values, in seconds
NET_TIMEOUT = 3
NET_RETRY_INTERVAL = 1
# SCM network-related values, in seconds
SCM_NET_TIMEOUT = 0.1
SCM_NET_RETRY_INTERVAL = 0.1
KOJI_CONFIG = "./conf/koji.conf"
KOJI_PROFILE = "staging"
SERVER_NAME = "localhost"
KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos"
SCMURLS = ["https://src.stg.fedoraproject.org/modules/"]
AUTH_METHOD = "oidc"
RESOLVER = "db"
ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"}
# Greenwave configuration
GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
GREENWAVE_DECISION_CONTEXT = "test_dec_context"
GREENWAVE_SUBJECT_TYPE = "some-module"
STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
class ProdConfiguration(BaseConfiguration):
pass
class LocalBuildConfiguration(BaseConfiguration):
CACHE_DIR = "~/modulebuild/cache"
LOG_LEVEL = "debug"
MESSAGING = "in_memory"
ARCH_AUTODETECT = True
ARCH_FALLBACK = "x86_64"
ALLOW_CUSTOM_SCMURLS = True
RESOLVER = "mbs"
RPMS_ALLOW_REPOSITORY = True
MODULES_ALLOW_REPOSITORY = True
class OfflineLocalBuildConfiguration(LocalBuildConfiguration):
RESOLVER = "local"
class DevConfiguration(LocalBuildConfiguration):
DEBUG = True
LOG_BACKEND = "console"
CELERY_BROKER_URL = "redis://localhost:6379/0"
CELERY_RESULT_BACKEND = "redis://localhost:6379/0"

70
conf/web_config.py
View File

@@ -1,70 +0,0 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from os import environ, path
confdir = path.abspath(path.dirname(__file__))
dbdir = path.abspath(path.join(confdir, "..")) if confdir.endswith("conf") else confdir
class WebConfiguration(object):
SECRET_KEY = "74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0"
SQLALCHEMY_DATABASE_URI = "sqlite:///{0}".format(path.join(dbdir, "module_build_service.db"))
SQLALCHEMY_TRACK_MODIFICATIONS = True
# Where we should run when running "manage.py run" directly.
HOST = "0.0.0.0"
PORT = 5000
class TestConfiguration(WebConfiguration):
LOG_LEVEL = "debug"
SQLALCHEMY_DATABASE_URI = environ.get(
"DATABASE_URI", "sqlite:///{0}".format(path.join(dbdir, "mbstest.db")))
DEBUG = True
MESSAGING = "in_memory"
# Global network-related values, in seconds
NET_TIMEOUT = 3
NET_RETRY_INTERVAL = 1
# SCM network-related values, in seconds
SCM_NET_TIMEOUT = 0.1
SCM_NET_RETRY_INTERVAL = 0.1
KOJI_CONFIG = "./conf/koji.conf"
KOJI_PROFILE = "staging"
SERVER_NAME = "localhost"
KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos"
SCMURLS = ["https://src.stg.fedoraproject.org/modules/"]
ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"}
# Greenwave configuration
GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
GREENWAVE_DECISION_CONTEXT = "test_dec_context"
GREENWAVE_SUBJECT_TYPE = "some-module"
STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
class ProdConfiguration(WebConfiguration):
pass
class LocalBuildConfiguration(WebConfiguration):
CACHE_DIR = "~/modulebuild/cache"
LOG_LEVEL = "debug"
MESSAGING = "in_memory"
ALLOW_CUSTOM_SCMURLS = True
RESOLVER = "mbs"
RPMS_ALLOW_REPOSITORY = True
MODULES_ALLOW_REPOSITORY = True
class OfflineLocalBuildConfiguration(LocalBuildConfiguration):
RESOLVER = "local"
class DevConfiguration(LocalBuildConfiguration):
DEBUG = True

9
module_build_service/__init__.py
View File

@@ -18,9 +18,7 @@ for a number of tasks:
infrastructure services can pick up the work.
"""
import os.path
import pkg_resources
import sys
from celery import Celery
from flask import Flask, has_app_context, url_for
from flask_sqlalchemy import SQLAlchemy
@@ -48,12 +46,7 @@ api_version = 2
app = Flask(__name__)
app.wsgi_app = ReverseProxy(app.wsgi_app)
backend_commands = ("fedmsg-hub", "celery", "build_module_locally")
if any([os.path.basename(arg).startswith(backend_commands) for arg in sys.argv]):
# running as backend
conf = init_config(app, backend=True)
else:
conf = init_config(app)
conf = init_config(app)
celery_app = Celery("module-build-service")
# Convert config names specific for Celery like this:

57
module_build_service/config.py
View File

@@ -24,19 +24,11 @@ SUPPORTED_RESOLVERS = {
}
def init_config(app, backend=False):
def init_config(app):
""" Configure MBS and the Flask app
By default, this create a config object with WebConfig, if backend is specified,
create a config object with BackendConfig.
"""
config_module = None
if backend:
config_module_name = "backend_config"
config_file = "/etc/module-build-service/backend_config.py"
else:
config_module_name = "web_config"
config_file = "/etc/module-build-service/web_config.py"
config_file = "/etc/module-build-service/config.py"
config_section = "DevConfiguration"
# automagically detect production environment:
@@ -74,14 +66,13 @@ def init_config(app, backend=False):
if "MBS_CONFIG_SECTION" in app.request.environ:
config_section = app.request.environ["MBS_CONFIG_SECTION"]
test_env = False
dev_env = False
true_options = ("1", "on", "true", "y", "yes")
# TestConfiguration shall only be used for running tests, otherwise...
if any(["py.test" in arg or "pytest" in arg for arg in sys.argv]):
test_env = True
config_section = "TestConfiguration"
from conf import config
config_module = config
# ...MODULE_BUILD_SERVICE_DEVELOPER_ENV has always the last word
# and overrides anything previously set before!
# Again, check Flask app (preferably) or fallback to os.environ.
@@ -89,16 +80,15 @@ def init_config(app, backend=False):
# -> /conf/config.py.
elif flask_app_env and "MODULE_BUILD_SERVICE_DEVELOPER_ENV" in app.request.environ:
if app.request.environ["MODULE_BUILD_SERVICE_DEVELOPER_ENV"].lower() in true_options:
dev_env = True
config_section = "DevConfiguration"
from conf import config
config_module = config
elif os.environ.get("MODULE_BUILD_SERVICE_DEVELOPER_ENV", "").lower() in true_options:
dev_env = True
config_section = "DevConfiguration"
from conf import config
if test_env or dev_env:
import importlib
config_module = importlib.import_module("conf.%s" % config_module_name)
config_module = config
# try loading configuration from file
if not config_module:
try:
@@ -108,10 +98,7 @@ def init_config(app, backend=False):
# finally configure MBS and the Flask app
config_section_obj = getattr(config_module, config_section)
if backend:
conf = BackendConfig(config_section_obj)
else:
conf = WebConfig(config_section_obj)
conf = Config(config_section_obj)
app.config.from_object(config_section_obj)
return conf
@@ -125,7 +112,7 @@ class Path:
class Config(object):
"""Class representing the orchestrator common configuration."""
"""Class representing the orchestrator configuration."""
_defaults = {
"debug": {"type": bool, "default": False, "desc": "Debug mode"},
@@ -934,23 +921,3 @@ class Config(object):
if i < 1:
raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1")
self._num_threads_for_build_submissions = i
class WebConfig(Config):
"""Class representing the orchestrator frontend web configuration."""
_web_defaults = {
}
def __init__(self, conf_section_obj):
self._defaults.update(self._web_defaults)
super(WebConfig, self).__init__(conf_section_obj)
class BackendConfig(Config):
"""Class representing the orchestrator backend workers configuration."""
_backend_defaults = {
}
def __init__(self, conf_section_obj):
self._defaults.update(self._backend_defaults)
super(BackendConfig, self).__init__(conf_section_obj)

2
module_build_service/models.py
View File

@@ -663,7 +663,7 @@ class ModuleBuild(MBSBase):
bus.
:param db_session: SQLAlchemy session object.
:param conf: MBS config object returned from function :func:`init_config`,
:param conf: MBS config object returned from function :func:`init_config`
which contains loaded configs.
:type conf: :class:`Config`
:param int state: the state value to transition to. Refer to ``BUILD_STATES``.

3
setup.py
View File

@@ -66,8 +66,7 @@ setup(
"/etc/module-build-service/",
[
"conf/cacert.pem",
"conf/web_config.py",
"conf/backend_config.py",
"conf/config.py",
"conf/koji.conf",
"conf/mock.cfg",
"conf/yum.conf",

2
tests/test_monitor.py
View File

@@ -8,7 +8,7 @@ import module_build_service.config as mbs_config
import module_build_service.monitor
from module_build_service import models
from module_build_service.db_session import db_session
from conf.web_config import TestConfiguration
from conf.config import TestConfiguration
from six.moves import reload_module
from tests import app, init_data, make_module_in_db