fedmsg endpoints

defaults/main.yml

@@ -10,7 +10,6 @@ mbs_mq_cert: ''
 mbs_mq_key: ''
 mbs_mq_ca: ''
 mbs_default_modules_dir: default-modules
-mbs_stomp_uri: ''
 # rabbitmq
 rabbitmq_local: true
 rabbitmq_vhost: mbs
@@ -29,22 +28,20 @@ mbs_config_secret_key: a1b2c3d4
 mbs_config_database_url: postgresql://mbs:mbs@localhost/mbs
 mbs_config_system: koji
 mbs_config_messaging: in_memory
-mbs_config_messaging_topic_prefix: 'org.centos.mbs'
+mbs_config_messaging_topic_prefix: ['org.centos.prod']
 mbs_config_koji_file: /etc/module-build-service/koji.conf
 mbs_config_koji_profile: koji
 mbs_config_arches:
   - x86_64
 mbs_config_koji_proxy_user: true
 mbs_config_koji_repository_url: https://kojipkgs.stg.fedoraproject.org/repos
-mbs_config_copr_file: /etc/module-build-service/copr.conf
-mbs_config_scm_urls:
-  - git+https://src.fedoraproject.org/modules/
+mbs_config_scm_urls: []
 mbs_config_polling_interval: 3600
-mbs_config_default_repository_url: 'git+https://src.fedoraproject.org/rpms/'
+mbs_config_default_repository_url: 'git+https://git.centos.org/rpms/'
 mbs_config_allow_rpm_repository: false
-mbs_config_rpm_default_cache_url: 'https://src.fedoraproject.org/repo/pkgs/'
+mbs_config_rpm_default_cache_url: 'https://git.centos.org/repo/pkgs/'
 mbs_config_rpm_allow_cache: false
-mbs_config_modules_repository_url: 'git+https://src.fedoraproject.org/modules/'
+mbs_config_modules_repository_url: 'git+https://git.centos.org/centos/modules'
 mbs_config_modules_allow_repository: false
 mbs_config_log_backend: file
 mbs_config_log_file: /var/log/mbs/module_build_service.log
@@ -52,19 +49,12 @@ mbs_config_log_level: info
 mbs_config_logs_build_dir: /var/tmp
 mbs_config_krb_keytab: /etc/krb5.mbs_mbs{{ mbs_config_env }}.centos.org.keytab
 mbs_config_krb_principal: mbs/mbs.centos.org@CENTOS.ORG
-mbs_config_krb_cache: /var/cache/fedmsg/mbs-krb5cc
-mbs_config_amq_recv_addreses:
-  - amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji
-  - amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service
-mbs_config_amq_dest_address: amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service
-mbs_config_amq_cert_file: /etc/module_build_service/msg-m8y-client.crt
-mbs_config_amq_key_file: /etc/module_build_service/msg-m8y-client.key
-mbs_config_amq_ca_file: /etc/module_build_service/Root-CA.crt
 mbs_config_allowed_groups:
   - packager
 mbs_config_admin_groups:
   - factory2
   - releng-team
+mbs_config_allowed_users: []
 mbs_config_rebuild_strategy: only-changed
 mbs_config_rebuild_strategy_override: true
 mbs_config_base_module_arches:
@@ -81,8 +71,6 @@ mbs_config_base_module_names:
 mbs_config_koji_build_tag: modular-updates-candidate
 mbs_config_koji_tag_prefixes:
   - module
-  - f26-modularity
-  - scrmod
 mbs_config_koji_tag_extra_opts:
   mock.package_manager: dnf
   repo_include_all: true
@@ -96,6 +84,27 @@ mbs_config_build_priority: 0
 mbs_config_koji_devel_module: false
 mbs_config_modules_allow_scratch: true
 mbs_config_modules_only_compatible: false
+mbs_koji_tag_permission: admin
+mbs_koji_cg_tag_build: true
+mbs_allow_name_override_from_scm: false
+mbs_allow_stream_override_from_scm: false
+mbs_num_concurrent_builds: 5
+mbs_allowed_privileged_module_names: []
+mbs_allowed_groups_to_import_module: []
+mbs_br_stream_override_module: "platform"
+mbs_br_stream_override_regexes: []
+mbs_product_pages_url: ''
+mbs_product_pages_schedule_task_name: ''
+mbs_product_pages_module_streams: {}
+mbs_scratch_build_only_branches: ["^private-.*"]
+mbs_fedmsg_endpoints: ["tcp://127.0.0.1:3001"]
+# fedora-messaging
+mbs_fedora_messaging_addrs: ["tcp://hub.fedoraproject.org:9940"]
+# relay
+mbs_relay_inbound_addrs: ["tcp://127.0.0.1:3999"]
+mbs_relay_outbound_addrs: ["tcp://127.0.0.1:9941"]
+# moksha
+mbs_moksha_monitoring_addr: tcp://0.0.0.0:10030
 # frontend
 mbs_wsgi_procs: 1
 mbs_wsgi_threads: 1
@@ -107,4 +116,8 @@ mbs_frontend_https_enabled: false
 mbs_frontend_https_port: 443
 mbs_frontend_http_enabled: true
 mbs_frontend_http_port: 80
-mbs_frontend_keytab: "{{ mbs_config_krb_keytab }}"
+mbs_frontend_auth_type: kerberos
+mbs_frontend_krb_keytab: ''
+mbs_frontend_krb_host: 'localhost'
+mbs_frontend_ldap_uri: ''
+mbs_frontend_ldap_dn_groups: ''

tasks/mbs-backend.yml

@@ -55,11 +55,18 @@
 
 - name: copy fedmsg configuration
   template:
-    src: mbs/mbs-fedmsg.py.j2
-    dest: /etc/fedmsg.d/mbs-fedmsg.py
+    src: mbs/{{ item }}.j2
+    dest: /etc/fedmsg.d/{{ item }}
     owner: root
     group: fedmsg
     mode: 0644
+  with_items:
+    - mbs-fedmsg.py
+    - mbs-logging.py
+    - logging.py
+    - mbs-endpoints.py
+    - endpoints.py
+    - relay.py
   notify:
     - Restart apache
     - Restart fedmsg-hub

templates/mbs/config.py.j2

@@ -1,22 +1,10 @@
-from os import path
-
-# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are
-# declared properly somewhere/somehow
-confdir = path.abspath(path.dirname(__file__))
-# use parent dir as dbdir else fallback to current dir
-dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \
-        else confdir
-
-
-class BaseConfiguration(object):
+class ProdConfiguration(object):
     DEBUG = {{ mbs_config_debug | bool }}
     # Make this random (used to generate session keys)
     SECRET_KEY = '{{ mbs_config_secret_key }}'
+
     SQLALCHEMY_DATABASE_URI = '{{ mbs_config_database_url }}'
     SQLALCHEMY_TRACK_MODIFICATIONS = True
-    # Where we should run when running "manage.py runssl" directly.
-    HOST = '0.0.0.0'
-    PORT = 5000
 
     CELERY_BROKER_URL = '{{ mbs_config_broker_url }}'
 
@@ -26,31 +14,73 @@ class BaseConfiguration(object):
 
     SYSTEM = '{{ mbs_config_system }}'
     MESSAGING = '{{ mbs_config_messaging }}'  # in_memory, fedmsg or amq
-    MESSAGING_TOPIC_PREFIX = ['{{ mbs_config_messaging_topic_prefix }}']
+    MESSAGING_TOPIC_PREFIX = {{ mbs_config_messaging_topic_prefix }}
     KOJI_CONFIG = '{{ mbs_config_koji_file }}'
     KOJI_PROFILE = '{{ mbs_config_koji_profile }}'
     ARCHES = {{ mbs_config_arches }}
     KOJI_PROXYUSER = {{ mbs_config_koji_proxy_user | bool }}
     KOJI_REPOSITORY_URL = '{{ mbs_config_koji_repository_url }}'
-    COPR_CONFIG = '{{ mbs_config_copr_file }}'
     SCMURLS = {{ mbs_config_scm_urls }}
 
+    RESOLVER = '{{ mbs_config_resolver }}'
+
+    # This is a whitelist of prefixes of koji tags we're allowed to manipulate
+    KOJI_TAG_PREFIXES = {{ mbs_config_koji_tag_prefixes }}
+
+    # Permission name to require for newly created Koji tags
+    KOJI_TAG_PERMISSION = '{{ mbs_koji_tag_permission }}'
+
+    # If this is too long, we could change it to 'fm_' some day.
+    DEFAULT_DIST_TAG_PREFIX = '{{ mbs_config_dist_tag_prefix }}'
+
+    # Relative Koji task priority (0 means default priority of 20).
+    KOJI_BUILD_PRIORITY = {{ mbs_config_build_priority }}
+
+    # Control where modules get tagged post-build. 
+    BASE_MODULE_NAMES = set({{ mbs_config_base_module_names }})
+    KOJI_CG_TAG_BUILD = {{ mbs_koji_cg_tag_build }}
+    KOJI_CG_BUILD_TAG_TEMPLATE = "{}-{{ mbs_config_koji_build_tag }}"
+    KOJI_CG_DEFAULT_BUILD_TAG = "{{ mbs_config_koji_build_tag }}"
+
+    # should always be false unless in development mode
+    NO_AUTH = {{ mbs_config_no_auth | bool }}
+
+    # Don't let people submit yaml directly.  it has to come from dist-git
+    YAML_SUBMIT_ALLOWED = {{ mbs_config_yaml_allowed | bool }}
+
+    # Allow maintainers to specify something that differs from the git branch.
+    ALLOW_NAME_OVERRIDE_FROM_SCM = {{ mbs_allow_name_override_from_scm }}
+    ALLOW_STREAM_OVERRIDE_FROM_SCM = {{ mbs_allow_stream_override_from_scm }}
+
     # How often should we resort to polling, in seconds
     # Set to zero to disable polling
     POLLING_INTERVAL = {{ mbs_config_polling_interval }}
 
+    # Determines how many builds that can be submitted to the builder
+    # and be in the build state at a time. Set this to 0 for no restrictions
+    # New name
+    NUM_CONCURRENT_BUILDS = {{ mbs_num_concurrent_builds }}
+
+    MODULES_ALLOW_SCRATCH = {{ mbs_config_modules_allow_scratch | bool }}
     RPMS_DEFAULT_REPOSITORY = '{{ mbs_config_default_repository_url }}'
     RPMS_ALLOW_REPOSITORY = {{ mbs_config_allow_rpm_repository | bool }}
     RPMS_DEFAULT_CACHE = '{{ mbs_config_rpm_default_cache_url }}'
     RPMS_ALLOW_CACHE = {{ mbs_config_rpm_allow_cache | bool }}
-
     MODULES_DEFAULT_REPOSITORY = '{{ mbs_config_modules_repository_url }}'
     MODULES_ALLOW_REPOSITORY = {{ mbs_config_modules_allow_repository | bool }}
 
+    # Our per-build logs for the koji-content generator go here.
+    # CG imports are controlled by KOJI_ENABLE_CONTENT_GENERATOR
+    BUILD_LOGS_DIR = '{{ mbs_config_logs_build_dir }}'
+
+    # Delete module-* targets one hour after build
+    KOJI_TARGET_DELETE_TIME = {{ mbs_config_koji_delete_time }}
+
+    # Whether or not to import modules back to koji.
+    KOJI_ENABLE_CONTENT_GENERATOR = {{ mbs_config_content_generator | bool }}
+
     # Available backends are: console, file, journal.
     LOG_BACKEND = '{{ mbs_config_log_backend }}'
-
-    # Path to log file when LOG_BACKEND is set to "file".
 {% if mbs_config_log_backend == 'file' %}
     LOG_FILE = '{{ mbs_config_log_file }}'
 {% endif %}
@@ -58,85 +88,52 @@ class BaseConfiguration(object):
     # Available log levels are: debug, info, warn, error.
     LOG_LEVEL = '{{ mbs_config_log_level }}'
 
-    # Settings for Kerberos
+    REBUILD_STRATEGY_ALLOW_OVERRIDE = {{ mbs_config_rebuild_strategy_override | bool }}
+    REBUILD_STRATEGY = '{{ mbs_config_rebuild_strategy }}'
+
+    ALLOWED_PRIVILEGED_MODULE_NAMES = {{ mbs_allowed_privileged_module_names }}
+
+    STREAM_SUFFIXES = {
+        r'el\d+\.\d+\.\d+\.z': 0.1,
+    }
+
+    #frontend
+    AUTH_METHOD = '{{ mbs_frontend_auth_type }}'
+    KERBEROS_KEYTAB = '{{ mbs_frontend_krb_keytab }}'
+    KERBEROS_HTTP_HOST = '{{ mbs_frontend_krb_host }}'
+    LDAP_URI = '{{ mbs_frontend_ldap_uri }}'
+    LDAP_GROUPS_DN = '{{ mbs_frontend_ldap_dn_groups }}'
+    ALLOWED_GROUPS = {{ mbs_config_allowed_groups }}
+    ALLOWED_USERS = {{ mbs_config_allowed_users }}
+    ADMIN_GROUPS = {{ mbs_config_admin_groups }}
+    ALLOWED_GROUPS_TO_IMPORT_MODULE = {{ mbs_allowed_groups_to_import_module }}
+    BR_STREAM_OVERRIDE_MODULE = '{{ mbs_br_stream_override_module }}'
+    BR_STREAM_OVERRIDE_REGEXES = [
+{% for regex in mbs_br_stream_override_regexes %}
+        r'{{ regex }}',
+{% endfor %}
+    ]
+
+    PRODUCT_PAGES_URL = '{{ mbs_product_pages_url }}'
+    PRODUCT_PAGES_SCHEDULE_TASK_NAME = '{{ mbs_product_pages_schedule_task_name }}'
+    PRODUCT_PAGES_MODULE_STREAMS = {
+{% for regex, values in mbs_product_pages_module_streams.items() %}
+        r'{{ regex }}': {{ values }},
+{% endfor %}
+    }
+
+    # Settings for Kerberos (backend)
     KRB_KEYTAB = '{{ mbs_config_krb_keytab }}'
     KRB_PRINCIPAL = '{{ mbs_config_krb_principal }}'
-    KRB_CCACHE = '{{ mbs_config_krb_cache }}'
 
-{% if mbs_config_messaging == 'amq' %}
-    # AMQ prefixed variables are required only while using 'amq' as messaging backend
-    # Addresses to listen to
-    AMQ_RECV_ADDRESSES = {{ mbs_config_amq_recv_addreses }}
-    # Address for sending messages
-    AMQ_DEST_ADDRESS = '{{ mbs_config_amq_dest_address }}'
-{% if mbs_config_amq_cert_file %}
-    AMQ_CERT_FILE = '{{ mbs_config_amq_cert_file }}'
-{% endif %}
-{% if mbs_config_amq_key_file %}
-    AMQ_PRIVATE_KEY_FILE = '{{ mbs_config_amq_key_file }}'
-{% endif %}
-{% if mbs_config_amq_ca_file %}
-    AMQ_TRUSTED_CERT_FILE = '{{ mbs_config_amq_ca_file }}'
-{% endif %}
-{% endif %}
-
-class ProdConfiguration(BaseConfiguration):
-    # These groups are allowed to submit builds.
-    ALLOWED_GROUPS = {{ mbs_config_allowed_groups }} # https://pagure.io/fesco/issue/1763
-
-    # These groups are allowed to cancel the builds of other users.
-    ADMIN_GROUPS = {{ mbs_config_admin_groups }}
-
-    REBUILD_STRATEGY = '{{ mbs_config_rebuild_strategy }}'
-    REBUILD_STRATEGY_ALLOW_OVERRIDE = {{ mbs_config_rebuild_strategy_override | bool }}
-
-    # Our per-build logs for the koji-content generator go here.
-    # CG imports are controlled by KOJI_ENABLE_CONTENT_GENERATOR
-    BUILD_LOGS_DIR = '{{ mbs_config_logs_build_dir }}'
-
-    BASE_MODULE_ARCHES = {{ mbs_config_base_module_arches }}
-
-    RESOLVER = '{{ mbs_config_resolver }}'
-
-    # Made possible by https://pagure.io/releng/issue/6799
-    KOJI_ENABLE_CONTENT_GENERATOR = {{ mbs_config_content_generator | bool }}
-
-    # See https://pagure.io/releng/issue/7012
-    BASE_MODULE_NAMES = set({{ mbs_config_base_module_names }})
-    KOJI_CG_BUILD_TAG_TEMPLATE = "{}-{{ mbs_config_koji_build_tag }}"
-    KOJI_CG_DEFAULT_BUILD_TAG = "{{ mbs_config_koji_build_tag }}"
-
-    # This is a whitelist of prefixes of koji tags we're allowed to manipulate
-    KOJI_TAG_PREFIXES = {{ mbs_config_koji_tag_prefixes }}
+    NUM_WORKERS = {{ mbs_num_workers }}
 
     # Extra options set for newly created Koji tags
     KOJI_TAG_EXTRA_OPTS = {{ mbs_config_koji_tag_extra_opts }}
 
-    # If this is too long, we could change it to 'fm_' some day.
-    DEFAULT_DIST_TAG_PREFIX = '{{ mbs_config_dist_tag_prefix }}'
-
-    # Delete module-* targets one hour after build
-    KOJI_TARGET_DELETE_TIME = {{ mbs_config_koji_delete_time }}
-
-    # yes, we want everyone to authenticate
-    NO_AUTH = {{ mbs_config_no_auth | bool }}  # Obviously.
-
-    # Don't let people submit yaml directly.  it has to come from dist-git
-    YAML_SUBMIT_ALLOWED = {{ mbs_config_yaml_allowed | bool }}
-
-    # Relative Koji task priority (0 means default priority of 20).
-    KOJI_BUILD_PRIORITY = {{ mbs_config_build_priority }}
-
-    # Koji Content Generator "-devel" modules aren't used in Fedora, so we can just disable them
-    KOJI_CG_DEVEL_MODULE = {{ mbs_config_koji_devel_module | bool }}
-
-    MODULES_ALLOW_SCRATCH = {{ mbs_config_modules_allow_scratch | bool }}
-
-    # By default, MBS allows buildrequiring only modules built against
-    # compatible version of platform base module. By compatible, we mean
-    # less or equal minor number of "stream_version". For example, when building module
-    # against platform:f30, it wouldn't be possible to buildrequire a module
-    # built against platform:f29. This is not intended behaviour in Fedora
-    # and therefore we want to turn this feature off.
-    ALLOW_ONLY_COMPATIBLE_BASE_MODULES = {{ mbs_config_modules_only_compatible | bool }}
+    SCRATCH_BUILD_ONLY_BRANCHES = [
+{% for regex in mbs_scratch_build_only_branches %}
+        r'{{ regex }}',
+{% endfor %}
+    ]
 

templates/mbs/endpoints.py.j2

@@ -0,0 +1,5 @@
+config = dict(
+  endpoints={
+    "supybot.value01": {{ mbs_fedora_messaging_addrs }}
+  }
+)

templates/mbs/logging.py.j2

@@ -0,0 +1,31 @@
+bare_format = "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s"
+
+config = dict(
+    logging=dict(
+        version=1,
+        formatters=dict(
+            bare={
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+                "format": bare_format
+            },
+        ),
+        handlers=dict(
+            console={
+                "class": "logging.StreamHandler",
+                "formatter": "bare",
+                "level": "{{ mbs_config_log_level|upper }}",
+                "stream": "ext://sys.stdout",
+            },
+        ),
+        loggers=dict(
+            fedmsg={
+                "level": "{{ mbs_config_log_level|upper }}",
+                "propagate": True,
+            },
+            moksha={
+                "level": "{{ mbs_config_log_level|upper }}",
+                "propagate": True,
+            },
+        ),
+    ),
+)

templates/mbs/mbs-endpoints.py.j2

@@ -0,0 +1,5 @@
+config = dict(
+    endpoints={
+        "mbs.mbs-backend01": {{ mbs_fedmsg_endpoints }}
+    },
+)

templates/mbs/mbs-fedmsg.py.j2

@@ -1,17 +1,7 @@
-import socket
-
-#config = {
-    # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
-    # 'cert_prefix': '{{ mbs_cert_prefix }}',
-    # 'name': 'mbs.%s' % socket.gethostname().split('.', 1)[0],
-#}
-
 config = {
-    'zmq_enabled': True,
-    'validate_signatures': False,
-    'endpoints': {},
-    'moksha.blocking_mode': True,
-    # moksha-monitor-exporter's point of contact
-    'moksha.monitoring.socket': 'tcp://0.0.0.0:10030',
+  'zmq_enabled': True,
+  'validate_signatures': False,
+  'moksha.blocking_mode': True,
+  # moksha-monitor-exporter's point of contact
+  'moksha.monitoring.socket': '{{ mbs_moksha_monitoring_addr }}',
 }
-

templates/mbs/mbs-logging.py.j2

@@ -0,0 +1,19 @@
+config = dict(
+    logging=dict(
+        loggers=dict(
+            # Quiet this guy down...
+            requests={
+                "level": "WARNING",
+                "propagate": True,
+            },
+            module_build_service={
+                "level": "{{ mbs_config_log_level|upper }}",
+                "propagate": True,
+            },
+        ),
+        root=dict(
+            handlers=["console"],
+            level="{{ mbs_config_log_level|upper }}",
+        ),
+    ),
+)

templates/mbs/mbs.conf.j2

@@ -75,8 +75,7 @@ WSGIScriptAlias / /usr/share/mbs/mbs.wsgi
 
   AuthType GSSAPI
   AuthName "MBS Kerberos negotiate authentication based on GSSAPI"
-  GssapiSSLonly On
-  GssapiCredStore keytab:{{ mbs_frontend_keytab }}
+  GssapiCredStore keytab:{{ mbs_frontend_krb_keytab }}
 
   # GET methods are allowed for everyone, but any other method
   # needs a valid-user.

templates/mbs/relay.py.j2

@@ -0,0 +1,8 @@
+config = dict(
+  endpoints={
+    # This is the output side of the relay to which all other
+    # services can listen.
+    "relay_outbound":{{ mbs_relay_inbound_addrs }}
+  },
+  relay_inbound={{ mbs_relay_outbound_addrs }}
+)