diff --git a/defaults/main.yml b/defaults/main.yml index 266009f..45b1bb5 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -10,7 +10,6 @@ mbs_mq_cert: '' mbs_mq_key: '' mbs_mq_ca: '' mbs_default_modules_dir: default-modules -mbs_stomp_uri: '' # rabbitmq rabbitmq_local: true rabbitmq_vhost: mbs @@ -29,22 +28,20 @@ mbs_config_secret_key: a1b2c3d4 mbs_config_database_url: postgresql://mbs:mbs@localhost/mbs mbs_config_system: koji mbs_config_messaging: in_memory -mbs_config_messaging_topic_prefix: 'org.centos.mbs' +mbs_config_messaging_topic_prefix: ['org.centos.prod'] mbs_config_koji_file: /etc/module-build-service/koji.conf mbs_config_koji_profile: koji mbs_config_arches: - x86_64 mbs_config_koji_proxy_user: true mbs_config_koji_repository_url: https://kojipkgs.stg.fedoraproject.org/repos -mbs_config_copr_file: /etc/module-build-service/copr.conf -mbs_config_scm_urls: - - git+https://src.fedoraproject.org/modules/ +mbs_config_scm_urls: [] mbs_config_polling_interval: 3600 -mbs_config_default_repository_url: 'git+https://src.fedoraproject.org/rpms/' +mbs_config_default_repository_url: 'git+https://git.centos.org/rpms/' mbs_config_allow_rpm_repository: false -mbs_config_rpm_default_cache_url: 'https://src.fedoraproject.org/repo/pkgs/' +mbs_config_rpm_default_cache_url: 'https://git.centos.org/repo/pkgs/' mbs_config_rpm_allow_cache: false -mbs_config_modules_repository_url: 'git+https://src.fedoraproject.org/modules/' +mbs_config_modules_repository_url: 'git+https://git.centos.org/centos/modules' mbs_config_modules_allow_repository: false mbs_config_log_backend: file mbs_config_log_file: /var/log/mbs/module_build_service.log @@ -52,19 +49,12 @@ mbs_config_log_level: info mbs_config_logs_build_dir: /var/tmp mbs_config_krb_keytab: /etc/krb5.mbs_mbs{{ mbs_config_env }}.centos.org.keytab mbs_config_krb_principal: mbs/mbs.centos.org@CENTOS.ORG -mbs_config_krb_cache: /var/cache/fedmsg/mbs-krb5cc -mbs_config_amq_recv_addreses: - - amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji - - amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service -mbs_config_amq_dest_address: amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service -mbs_config_amq_cert_file: /etc/module_build_service/msg-m8y-client.crt -mbs_config_amq_key_file: /etc/module_build_service/msg-m8y-client.key -mbs_config_amq_ca_file: /etc/module_build_service/Root-CA.crt mbs_config_allowed_groups: - packager mbs_config_admin_groups: - factory2 - releng-team +mbs_config_allowed_users: [] mbs_config_rebuild_strategy: only-changed mbs_config_rebuild_strategy_override: true mbs_config_base_module_arches: @@ -81,8 +71,6 @@ mbs_config_base_module_names: mbs_config_koji_build_tag: modular-updates-candidate mbs_config_koji_tag_prefixes: - module - - f26-modularity - - scrmod mbs_config_koji_tag_extra_opts: mock.package_manager: dnf repo_include_all: true @@ -96,6 +84,27 @@ mbs_config_build_priority: 0 mbs_config_koji_devel_module: false mbs_config_modules_allow_scratch: true mbs_config_modules_only_compatible: false +mbs_koji_tag_permission: admin +mbs_koji_cg_tag_build: true +mbs_allow_name_override_from_scm: false +mbs_allow_stream_override_from_scm: false +mbs_num_concurrent_builds: 5 +mbs_allowed_privileged_module_names: [] +mbs_allowed_groups_to_import_module: [] +mbs_br_stream_override_module: "platform" +mbs_br_stream_override_regexes: [] +mbs_product_pages_url: '' +mbs_product_pages_schedule_task_name: '' +mbs_product_pages_module_streams: {} +mbs_scratch_build_only_branches: ["^private-.*"] +mbs_fedmsg_endpoints: ["tcp://127.0.0.1:3001"] +# fedora-messaging +mbs_fedora_messaging_addrs: ["tcp://hub.fedoraproject.org:9940"] +# relay +mbs_relay_inbound_addrs: ["tcp://127.0.0.1:3999"] +mbs_relay_outbound_addrs: ["tcp://127.0.0.1:9941"] +# moksha +mbs_moksha_monitoring_addr: tcp://0.0.0.0:10030 # frontend mbs_wsgi_procs: 1 mbs_wsgi_threads: 1 @@ -107,4 +116,8 @@ mbs_frontend_https_enabled: false mbs_frontend_https_port: 443 mbs_frontend_http_enabled: true mbs_frontend_http_port: 80 -mbs_frontend_keytab: "{{ mbs_config_krb_keytab }}" \ No newline at end of file +mbs_frontend_auth_type: kerberos +mbs_frontend_krb_keytab: '' +mbs_frontend_krb_host: 'localhost' +mbs_frontend_ldap_uri: '' +mbs_frontend_ldap_dn_groups: '' \ No newline at end of file diff --git a/tasks/mbs-backend.yml b/tasks/mbs-backend.yml index 80ecc03..480735b 100644 --- a/tasks/mbs-backend.yml +++ b/tasks/mbs-backend.yml @@ -55,11 +55,18 @@ - name: copy fedmsg configuration template: - src: mbs/mbs-fedmsg.py.j2 - dest: /etc/fedmsg.d/mbs-fedmsg.py + src: mbs/{{ item }}.j2 + dest: /etc/fedmsg.d/{{ item }} owner: root group: fedmsg mode: 0644 + with_items: + - mbs-fedmsg.py + - mbs-logging.py + - logging.py + - mbs-endpoints.py + - endpoints.py + - relay.py notify: - Restart apache - Restart fedmsg-hub diff --git a/templates/mbs/config.py.j2 b/templates/mbs/config.py.j2 index b06f550..d96b75d 100644 --- a/templates/mbs/config.py.j2 +++ b/templates/mbs/config.py.j2 @@ -1,22 +1,10 @@ -from os import path - -# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are -# declared properly somewhere/somehow -confdir = path.abspath(path.dirname(__file__)) -# use parent dir as dbdir else fallback to current dir -dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \ - else confdir - - -class BaseConfiguration(object): +class ProdConfiguration(object): DEBUG = {{ mbs_config_debug | bool }} # Make this random (used to generate session keys) SECRET_KEY = '{{ mbs_config_secret_key }}' + SQLALCHEMY_DATABASE_URI = '{{ mbs_config_database_url }}' SQLALCHEMY_TRACK_MODIFICATIONS = True - # Where we should run when running "manage.py runssl" directly. - HOST = '0.0.0.0' - PORT = 5000 CELERY_BROKER_URL = '{{ mbs_config_broker_url }}' @@ -26,31 +14,73 @@ class BaseConfiguration(object): SYSTEM = '{{ mbs_config_system }}' MESSAGING = '{{ mbs_config_messaging }}' # in_memory, fedmsg or amq - MESSAGING_TOPIC_PREFIX = ['{{ mbs_config_messaging_topic_prefix }}'] + MESSAGING_TOPIC_PREFIX = {{ mbs_config_messaging_topic_prefix }} KOJI_CONFIG = '{{ mbs_config_koji_file }}' KOJI_PROFILE = '{{ mbs_config_koji_profile }}' ARCHES = {{ mbs_config_arches }} KOJI_PROXYUSER = {{ mbs_config_koji_proxy_user | bool }} KOJI_REPOSITORY_URL = '{{ mbs_config_koji_repository_url }}' - COPR_CONFIG = '{{ mbs_config_copr_file }}' SCMURLS = {{ mbs_config_scm_urls }} + RESOLVER = '{{ mbs_config_resolver }}' + + # This is a whitelist of prefixes of koji tags we're allowed to manipulate + KOJI_TAG_PREFIXES = {{ mbs_config_koji_tag_prefixes }} + + # Permission name to require for newly created Koji tags + KOJI_TAG_PERMISSION = '{{ mbs_koji_tag_permission }}' + + # If this is too long, we could change it to 'fm_' some day. + DEFAULT_DIST_TAG_PREFIX = '{{ mbs_config_dist_tag_prefix }}' + + # Relative Koji task priority (0 means default priority of 20). + KOJI_BUILD_PRIORITY = {{ mbs_config_build_priority }} + + # Control where modules get tagged post-build. + BASE_MODULE_NAMES = set({{ mbs_config_base_module_names }}) + KOJI_CG_TAG_BUILD = {{ mbs_koji_cg_tag_build }} + KOJI_CG_BUILD_TAG_TEMPLATE = "{}-{{ mbs_config_koji_build_tag }}" + KOJI_CG_DEFAULT_BUILD_TAG = "{{ mbs_config_koji_build_tag }}" + + # should always be false unless in development mode + NO_AUTH = {{ mbs_config_no_auth | bool }} + + # Don't let people submit yaml directly. it has to come from dist-git + YAML_SUBMIT_ALLOWED = {{ mbs_config_yaml_allowed | bool }} + + # Allow maintainers to specify something that differs from the git branch. + ALLOW_NAME_OVERRIDE_FROM_SCM = {{ mbs_allow_name_override_from_scm }} + ALLOW_STREAM_OVERRIDE_FROM_SCM = {{ mbs_allow_stream_override_from_scm }} + # How often should we resort to polling, in seconds # Set to zero to disable polling POLLING_INTERVAL = {{ mbs_config_polling_interval }} + # Determines how many builds that can be submitted to the builder + # and be in the build state at a time. Set this to 0 for no restrictions + # New name + NUM_CONCURRENT_BUILDS = {{ mbs_num_concurrent_builds }} + + MODULES_ALLOW_SCRATCH = {{ mbs_config_modules_allow_scratch | bool }} RPMS_DEFAULT_REPOSITORY = '{{ mbs_config_default_repository_url }}' RPMS_ALLOW_REPOSITORY = {{ mbs_config_allow_rpm_repository | bool }} RPMS_DEFAULT_CACHE = '{{ mbs_config_rpm_default_cache_url }}' RPMS_ALLOW_CACHE = {{ mbs_config_rpm_allow_cache | bool }} - MODULES_DEFAULT_REPOSITORY = '{{ mbs_config_modules_repository_url }}' MODULES_ALLOW_REPOSITORY = {{ mbs_config_modules_allow_repository | bool }} + # Our per-build logs for the koji-content generator go here. + # CG imports are controlled by KOJI_ENABLE_CONTENT_GENERATOR + BUILD_LOGS_DIR = '{{ mbs_config_logs_build_dir }}' + + # Delete module-* targets one hour after build + KOJI_TARGET_DELETE_TIME = {{ mbs_config_koji_delete_time }} + + # Whether or not to import modules back to koji. + KOJI_ENABLE_CONTENT_GENERATOR = {{ mbs_config_content_generator | bool }} + # Available backends are: console, file, journal. LOG_BACKEND = '{{ mbs_config_log_backend }}' - - # Path to log file when LOG_BACKEND is set to "file". {% if mbs_config_log_backend == 'file' %} LOG_FILE = '{{ mbs_config_log_file }}' {% endif %} @@ -58,85 +88,52 @@ class BaseConfiguration(object): # Available log levels are: debug, info, warn, error. LOG_LEVEL = '{{ mbs_config_log_level }}' - # Settings for Kerberos + REBUILD_STRATEGY_ALLOW_OVERRIDE = {{ mbs_config_rebuild_strategy_override | bool }} + REBUILD_STRATEGY = '{{ mbs_config_rebuild_strategy }}' + + ALLOWED_PRIVILEGED_MODULE_NAMES = {{ mbs_allowed_privileged_module_names }} + + STREAM_SUFFIXES = { + r'el\d+\.\d+\.\d+\.z': 0.1, + } + + #frontend + AUTH_METHOD = '{{ mbs_frontend_auth_type }}' + KERBEROS_KEYTAB = '{{ mbs_frontend_krb_keytab }}' + KERBEROS_HTTP_HOST = '{{ mbs_frontend_krb_host }}' + LDAP_URI = '{{ mbs_frontend_ldap_uri }}' + LDAP_GROUPS_DN = '{{ mbs_frontend_ldap_dn_groups }}' + ALLOWED_GROUPS = {{ mbs_config_allowed_groups }} + ALLOWED_USERS = {{ mbs_config_allowed_users }} + ADMIN_GROUPS = {{ mbs_config_admin_groups }} + ALLOWED_GROUPS_TO_IMPORT_MODULE = {{ mbs_allowed_groups_to_import_module }} + BR_STREAM_OVERRIDE_MODULE = '{{ mbs_br_stream_override_module }}' + BR_STREAM_OVERRIDE_REGEXES = [ +{% for regex in mbs_br_stream_override_regexes %} + r'{{ regex }}', +{% endfor %} + ] + + PRODUCT_PAGES_URL = '{{ mbs_product_pages_url }}' + PRODUCT_PAGES_SCHEDULE_TASK_NAME = '{{ mbs_product_pages_schedule_task_name }}' + PRODUCT_PAGES_MODULE_STREAMS = { +{% for regex, values in mbs_product_pages_module_streams.items() %} + r'{{ regex }}': {{ values }}, +{% endfor %} + } + + # Settings for Kerberos (backend) KRB_KEYTAB = '{{ mbs_config_krb_keytab }}' KRB_PRINCIPAL = '{{ mbs_config_krb_principal }}' - KRB_CCACHE = '{{ mbs_config_krb_cache }}' -{% if mbs_config_messaging == 'amq' %} - # AMQ prefixed variables are required only while using 'amq' as messaging backend - # Addresses to listen to - AMQ_RECV_ADDRESSES = {{ mbs_config_amq_recv_addreses }} - # Address for sending messages - AMQ_DEST_ADDRESS = '{{ mbs_config_amq_dest_address }}' -{% if mbs_config_amq_cert_file %} - AMQ_CERT_FILE = '{{ mbs_config_amq_cert_file }}' -{% endif %} -{% if mbs_config_amq_key_file %} - AMQ_PRIVATE_KEY_FILE = '{{ mbs_config_amq_key_file }}' -{% endif %} -{% if mbs_config_amq_ca_file %} - AMQ_TRUSTED_CERT_FILE = '{{ mbs_config_amq_ca_file }}' -{% endif %} -{% endif %} - -class ProdConfiguration(BaseConfiguration): - # These groups are allowed to submit builds. - ALLOWED_GROUPS = {{ mbs_config_allowed_groups }} # https://pagure.io/fesco/issue/1763 - - # These groups are allowed to cancel the builds of other users. - ADMIN_GROUPS = {{ mbs_config_admin_groups }} - - REBUILD_STRATEGY = '{{ mbs_config_rebuild_strategy }}' - REBUILD_STRATEGY_ALLOW_OVERRIDE = {{ mbs_config_rebuild_strategy_override | bool }} - - # Our per-build logs for the koji-content generator go here. - # CG imports are controlled by KOJI_ENABLE_CONTENT_GENERATOR - BUILD_LOGS_DIR = '{{ mbs_config_logs_build_dir }}' - - BASE_MODULE_ARCHES = {{ mbs_config_base_module_arches }} - - RESOLVER = '{{ mbs_config_resolver }}' - - # Made possible by https://pagure.io/releng/issue/6799 - KOJI_ENABLE_CONTENT_GENERATOR = {{ mbs_config_content_generator | bool }} - - # See https://pagure.io/releng/issue/7012 - BASE_MODULE_NAMES = set({{ mbs_config_base_module_names }}) - KOJI_CG_BUILD_TAG_TEMPLATE = "{}-{{ mbs_config_koji_build_tag }}" - KOJI_CG_DEFAULT_BUILD_TAG = "{{ mbs_config_koji_build_tag }}" - - # This is a whitelist of prefixes of koji tags we're allowed to manipulate - KOJI_TAG_PREFIXES = {{ mbs_config_koji_tag_prefixes }} + NUM_WORKERS = {{ mbs_num_workers }} # Extra options set for newly created Koji tags KOJI_TAG_EXTRA_OPTS = {{ mbs_config_koji_tag_extra_opts }} - # If this is too long, we could change it to 'fm_' some day. - DEFAULT_DIST_TAG_PREFIX = '{{ mbs_config_dist_tag_prefix }}' - - # Delete module-* targets one hour after build - KOJI_TARGET_DELETE_TIME = {{ mbs_config_koji_delete_time }} - - # yes, we want everyone to authenticate - NO_AUTH = {{ mbs_config_no_auth | bool }} # Obviously. - - # Don't let people submit yaml directly. it has to come from dist-git - YAML_SUBMIT_ALLOWED = {{ mbs_config_yaml_allowed | bool }} - - # Relative Koji task priority (0 means default priority of 20). - KOJI_BUILD_PRIORITY = {{ mbs_config_build_priority }} - - # Koji Content Generator "-devel" modules aren't used in Fedora, so we can just disable them - KOJI_CG_DEVEL_MODULE = {{ mbs_config_koji_devel_module | bool }} - - MODULES_ALLOW_SCRATCH = {{ mbs_config_modules_allow_scratch | bool }} - - # By default, MBS allows buildrequiring only modules built against - # compatible version of platform base module. By compatible, we mean - # less or equal minor number of "stream_version". For example, when building module - # against platform:f30, it wouldn't be possible to buildrequire a module - # built against platform:f29. This is not intended behaviour in Fedora - # and therefore we want to turn this feature off. - ALLOW_ONLY_COMPATIBLE_BASE_MODULES = {{ mbs_config_modules_only_compatible | bool }} + SCRATCH_BUILD_ONLY_BRANCHES = [ +{% for regex in mbs_scratch_build_only_branches %} + r'{{ regex }}', +{% endfor %} + ] diff --git a/templates/mbs/endpoints.py.j2 b/templates/mbs/endpoints.py.j2 new file mode 100644 index 0000000..f8dcb63 --- /dev/null +++ b/templates/mbs/endpoints.py.j2 @@ -0,0 +1,5 @@ +config = dict( + endpoints={ + "supybot.value01": {{ mbs_fedora_messaging_addrs }} + } +) diff --git a/templates/mbs/logging.py.j2 b/templates/mbs/logging.py.j2 new file mode 100644 index 0000000..b57a54a --- /dev/null +++ b/templates/mbs/logging.py.j2 @@ -0,0 +1,31 @@ +bare_format = "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s" + +config = dict( + logging=dict( + version=1, + formatters=dict( + bare={ + "datefmt": "%Y-%m-%d %H:%M:%S", + "format": bare_format + }, + ), + handlers=dict( + console={ + "class": "logging.StreamHandler", + "formatter": "bare", + "level": "{{ mbs_config_log_level|upper }}", + "stream": "ext://sys.stdout", + }, + ), + loggers=dict( + fedmsg={ + "level": "{{ mbs_config_log_level|upper }}", + "propagate": True, + }, + moksha={ + "level": "{{ mbs_config_log_level|upper }}", + "propagate": True, + }, + ), + ), +) diff --git a/templates/mbs/mbs-endpoints.py.j2 b/templates/mbs/mbs-endpoints.py.j2 new file mode 100644 index 0000000..05ad7b9 --- /dev/null +++ b/templates/mbs/mbs-endpoints.py.j2 @@ -0,0 +1,5 @@ +config = dict( + endpoints={ + "mbs.mbs-backend01": {{ mbs_fedmsg_endpoints }} + }, +) \ No newline at end of file diff --git a/templates/mbs/mbs-fedmsg.py.j2 b/templates/mbs/mbs-fedmsg.py.j2 index 3e888b1..80f96be 100644 --- a/templates/mbs/mbs-fedmsg.py.j2 +++ b/templates/mbs/mbs-fedmsg.py.j2 @@ -1,17 +1,7 @@ -import socket - -#config = { - # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py - # 'cert_prefix': '{{ mbs_cert_prefix }}', - # 'name': 'mbs.%s' % socket.gethostname().split('.', 1)[0], -#} - config = { - 'zmq_enabled': True, - 'validate_signatures': False, - 'endpoints': {}, - 'moksha.blocking_mode': True, - # moksha-monitor-exporter's point of contact - 'moksha.monitoring.socket': 'tcp://0.0.0.0:10030', + 'zmq_enabled': True, + 'validate_signatures': False, + 'moksha.blocking_mode': True, + # moksha-monitor-exporter's point of contact + 'moksha.monitoring.socket': '{{ mbs_moksha_monitoring_addr }}', } - diff --git a/templates/mbs/mbs-logging.py.j2 b/templates/mbs/mbs-logging.py.j2 new file mode 100644 index 0000000..3913566 --- /dev/null +++ b/templates/mbs/mbs-logging.py.j2 @@ -0,0 +1,19 @@ +config = dict( + logging=dict( + loggers=dict( + # Quiet this guy down... + requests={ + "level": "WARNING", + "propagate": True, + }, + module_build_service={ + "level": "{{ mbs_config_log_level|upper }}", + "propagate": True, + }, + ), + root=dict( + handlers=["console"], + level="{{ mbs_config_log_level|upper }}", + ), + ), +) diff --git a/templates/mbs/mbs.conf.j2 b/templates/mbs/mbs.conf.j2 index 2009e83..9b9484f 100644 --- a/templates/mbs/mbs.conf.j2 +++ b/templates/mbs/mbs.conf.j2 @@ -75,8 +75,7 @@ WSGIScriptAlias / /usr/share/mbs/mbs.wsgi AuthType GSSAPI AuthName "MBS Kerberos negotiate authentication based on GSSAPI" - GssapiSSLonly On - GssapiCredStore keytab:{{ mbs_frontend_keytab }} + GssapiCredStore keytab:{{ mbs_frontend_krb_keytab }} # GET methods are allowed for everyone, but any other method # needs a valid-user. diff --git a/templates/mbs/relay.py.j2 b/templates/mbs/relay.py.j2 new file mode 100644 index 0000000..91e23b6 --- /dev/null +++ b/templates/mbs/relay.py.j2 @@ -0,0 +1,8 @@ +config = dict( + endpoints={ + # This is the output side of the relay to which all other + # services can listen. + "relay_outbound":{{ mbs_relay_inbound_addrs }} + }, + relay_inbound={{ mbs_relay_outbound_addrs }} +)