diff --git a/inventory/group_vars/check-compose b/inventory/group_vars/check-compose new file mode 100644 index 0000000000..568d81cb2b --- /dev/null +++ b/inventory/group_vars/check-compose @@ -0,0 +1,15 @@ +# This var should never be set for more than one machine +checkcompose_prod: true +checkcompose_env_suffix: +checkcompose_env: production + +# The checkcompose settings below cause system(s) in this group to +# send out check-compose reports. This could cause duplicate reports +# if additional systems were added to this group. +checkcompose_emailfrom: rawhide@fedoraproject.org +checkcompose_emailto: "test@lists.fedoraproject.org devel@lists.fedoraproject.org" +checkcompose_atomic_emailto: "dusty@dustymabe.com walters@verbum.org atomic@lists.fedoraproject.org" +checkcompose_atomic_emailerror: "true" +checkcompose_smtp: bastion.phx2.fedoraproject.org +checkcompose_url: "https://{{ external_hostname }}" +checkcompose_greenwaveurl: https://greenwave-web-greenwave.app.os.fedoraproject.org diff --git a/inventory/group_vars/check-compose_common b/inventory/group_vars/check-compose_common new file mode 100644 index 0000000000..bb999faf14 --- /dev/null +++ b/inventory/group_vars/check-compose_common @@ -0,0 +1,17 @@ +# we need this for our fedora-messaging consumer as it is not allowed +# to create queues on the infra AMQP broker, by broker config +checkcompose_amqp_passive: true + +# fedora-messaging job scheduler settings: most of these are the same +# for prod and stg as they both must listen for prod messages. Only +# the queue names differs +checkcompose_amqp_url: "amqps://openqa:@rabbitmq.fedoraproject.org/%2Fpubsub" +checkcompose_amqp_cacert: /etc/fedora-messaging/cacert.pem +checkcompose_amqp_key: /etc/pki/fedora-messaging/openqa-key.pem +checkcompose_amqp_cert: /etc/pki/fedora-messaging/openqa-cert.pem +checkcompose_amqp_queue: "openqa_checkcomp{{ checkcompose_env_suffix }}" +checkcompose_amqp_routing_keys: ["org.fedoraproject.{{ deployment_type }}.openqa.job.done"] + +# fedora-messaging email error reporting settings +checkcompose_amqp_mailto: ["adamwill@fedoraproject.org", "lruzicka@fedoraproject.org"] +checkcompose_amqp_smtp: bastion diff --git a/inventory/group_vars/check-compose_stg b/inventory/group_vars/check-compose_stg new file mode 100644 index 0000000000..1c57929a07 --- /dev/null +++ b/inventory/group_vars/check-compose_stg @@ -0,0 +1,6 @@ +checkcompose_prod: false +checkcompose_env_suffix: .stg +checkcompose_env: staging + +checkcompose_url: "https://{{ external_hostname }}" +checkcompose_greenwaveurl: https://greenwave-web-greenwave.app.os.stg.fedoraproject.org diff --git a/inventory/group_vars/openqa b/inventory/group_vars/openqa index 5bb89ba5e4..fc554e4c24 100644 --- a/inventory/group_vars/openqa +++ b/inventory/group_vars/openqa @@ -22,16 +22,5 @@ wikitcms_token: "{{ private }}/files/openidc/production/wikitcms.json" openqa_wikitcms_hostname: fedoraproject.org openqa_resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/ -# The checkcompose settings below cause system(s) in this group to -# send out check-compose reports. This could cause duplicate reports -# if additional systems were added to this group. -checkcompose_emailfrom: rawhide@fedoraproject.org -checkcompose_emailto: "test@lists.fedoraproject.org devel@lists.fedoraproject.org" -checkcompose_atomic_emailto: "dusty@dustymabe.com walters@verbum.org atomic@lists.fedoraproject.org" -checkcompose_atomic_emailerror: "true" -checkcompose_smtp: bastion.phx2.fedoraproject.org -checkcompose_url: "https://{{ external_hostname }}" -checkcompose_greenwaveurl: https://greenwave-web-greenwave.app.os.fedoraproject.org - deployment_type: prod freezes: false diff --git a/inventory/group_vars/openqa_stg b/inventory/group_vars/openqa_stg index 25e236f63d..15d8f18f27 100644 --- a/inventory/group_vars/openqa_stg +++ b/inventory/group_vars/openqa_stg @@ -43,9 +43,6 @@ openqa_scheduler_uuid: 47acfdb9-e323-4a13-a940-fdb903934e60 openqa_wiki_reporter_uuid: 2aa0c3fc-905d-4e59-8557-bfa23fbad28e openqa_resultsdb_reporter_uuid: aaf0f9b2-6858-4694-b018-356342761bc0 -checkcompose_url: "https://{{ external_hostname }}" -checkcompose_greenwaveurl: https://greenwave-web-greenwave.app.os.stg.fedoraproject.org - deployment_type: stg freezes: false diff --git a/inventory/inventory b/inventory/inventory index c5fefc140a..c113a2dc73 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -677,6 +677,17 @@ openqa01.qa.fedoraproject.org [relvalconsumer_test] openqa-stg01.qa.fedoraproject.org +[check-compose_common] +openqa01.qa.fedoraproject.org +openqa-stg01.qa.fedoraproject.org + +# This group should only ever contain *ONE* system +[check-compose] +openqa01.qa.fedoraproject.org + +[check-compose_stg] +openqa-stg01.qa.fedoraproject.org + # This group should only ever contain *ONE* system # It should probably be autocloud itself, but I dunno if that box # is authorized for ResultsDB currently diff --git a/playbooks/groups/openqa.yml b/playbooks/groups/openqa.yml index 5ede3c0d78..69c3a759ab 100644 --- a/playbooks/groups/openqa.yml +++ b/playbooks/groups/openqa.yml @@ -91,6 +91,19 @@ env: "production" env_suffix: "" tags: ['rabbit'] + # check-compose queue, we still use openqa auth for this as it's + # easy that way + - role: rabbit/queue + username: "openqa{{ openqa_env_suffix }}" + queue_name: "openqa_checkcomp{{ checkcompose_env_suffix }}" + routing_keys: + - "org.fedoraproject.{{ deployment_type }}.openqa.job.done" + vars: + # for this role to work correctly we do need to look like a + # staging server + env: "{{ checkcompose_env }}" + env_suffix: "{{ checkcompose_env_suffix }}" + tags: ['rabbit'] - name: set up openQA server data NFS mounts (staging) hosts: openqa_stg diff --git a/roles/check-compose/defaults/main.yml b/roles/check-compose/defaults/main.yml new file mode 100644 index 0000000000..36dc362fed --- /dev/null +++ b/roles/check-compose/defaults/main.yml @@ -0,0 +1,9 @@ +checkcompose_prod: false +checkcompose_amqp_passive: false +checkcompose_amqp_url: "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" +checkcompose_amqp_cacert: /etc/fedora-messaging/cacert.pem +checkcompose_amqp_key: /etc/fedora-messaging/fedora-key.pem +checkcompose_amqp_cert: /etc/fedora-messaging/fedora-cert.pem +checkcompose_amqp_routing_keys: ["org.fedoraproject.prod.openqa.job.done"] +checkcompose_amqp_mailfrom: "root@{{ external_hostname }}" +checkcompose_amqp_smtp: localhost diff --git a/roles/check-compose/handlers/main.yml b/roles/check-compose/handlers/main.yml new file mode 100644 index 0000000000..5571a63dc3 --- /dev/null +++ b/roles/check-compose/handlers/main.yml @@ -0,0 +1,5 @@ +# Restart handler for our fedora-messaging consumer +- name: Conditionally restart check-compose consumer service + command: /usr/local/bin/conditional-restart.sh fm-consumer@check-compose + listen: + - restart check-compose diff --git a/roles/check-compose/tasks/main.yml b/roles/check-compose/tasks/main.yml index 0ef18a8382..1514fdd3bb 100644 --- a/roles/check-compose/tasks/main.yml +++ b/roles/check-compose/tasks/main.yml @@ -1,3 +1,58 @@ +# Required vars +# - checkcompose_amqp_queue +## string - Message queue name for the consumer. To use the +## fedora-messaging scheduler with public authentication +## on the Fedora production AMQP broker (which is what +## you'd typically want), you only need to set this. +## This should be a unique and private string; the +## official recommendation is to use a random UUID +## generated by uuidgen. +# +# Required vars with defaults +# - checkcompose_prod +## bool - whether this is the production instance. **SHOULD ONLY +## EVER BE TRUE ON ONE SYSTEM IN THE WORLD** +## default - False +# - checkcompose_amqp_passive +## bool - If true, passive_declares will be set true in all the +## fedora-messaging consumer configuration files. This +## is needed for private authentication on the Fedora +## brokers. +## default - False +# - checkcompose_amqp_url +## string - AMQP broker URL for fedora-messaging report sender. +## The role default for this is the Fedora production +## broker with the shared 'fedora' username. +# - checkcompose_amqp_cacert +## string - CA certificate file to use for authenticating with +## AMQP broker for fedora-messaging report sender. +## The role default for this is the CA cert file for the +## Fedora production broker. +# - checkcompose_amqp_cert +## string - Certificate file to use for authenticating with AMQP +## broker for fedora-messaging report sender. The role +## default for this is the certificate file for the +## public 'fedora' account on the Fedora production +## broker. +# - checkcompose_amqp_key +## string - Private key file to use for authenticating with AMQP +## broker for fedora-messaging report sender. The role +## default for this is the key file for the public +## 'fedora' account on the Fedora production broker. +# - checkcompose_amqp_routing_keys +## list - List of routing key names for the fedora-messaging +## scheduler to subscribe to. The role default for this +## is the appropriate keys for the Fedora production +## broker. +# - checkcompose_amqp_mailfrom +## string - From email address for error report emails. Defaults +## to "root@{{ external_hostname }}". Only relevant if +## checkcompose_amqp_mailto is set. +# - checkcompose_amqp_smtp +## string - Hostname of SMTP server to use for sending consumer error +## emails. Defaults to 'localhost'. Only relevant if +## checkcompose_amqp_mailto is set. +# # Optional vars # - checkcompose_server ## string - hostname of openQA server to connect to (if not set, openQA @@ -27,6 +82,9 @@ ## string - File to append JSON-formatted report summary to # - checkcompose_loglevel ## string - log level +# - checkcompose_amqp_mailto +## list - List of email addresses to email errors to. If set, +## the email log handler will be configured. # # NOTE: this is missing /etc/openqa/client.conf configuration, for now; # we're assuming it'll be deployed on the openqa server boxes and hence @@ -44,8 +102,7 @@ - name: Install required packages package: - name: ['python3-fedfind', 'python3-fedmsg', 'python3-openqa_client', - 'python3-setuptools', 'python3-six'] + name: ['fedora-messaging', 'python3-fedfind', 'python3-openqa_client', 'python3-setuptools'] state: present tags: - packages @@ -62,10 +119,48 @@ chdir: /root/check-compose when: "gitcc is changed" notify: - - restart fedmsg-hub + - restart check-compose -- name: Enable fedmsg consumer - template: src=checkcomp_consumer.py.j2 dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644 +- name: Create /etc/pki/fedora-messaging + file: + dest: /etc/pki/fedora-messaging + mode: 0775 + owner: root + group: root + state: directory + when: "deployment_type is defined" + tags: + - config + +- name: Deploy the Fedora infra fedora-messaging cert + copy: + src: "{{ private }}/files/rabbitmq/{{checkcompose_env}}/pki/issued/openqa{{checkcompose_env_suffix}}.crt" + dest: /etc/pki/fedora-messaging/openqa{{checkcompose_env_suffix}}-cert.pem + mode: 0644 + owner: root + group: root + tags: + - config + +- name: Deploy the Fedora infra fedora-messaging key + copy: + src: "{{ private }}/files/rabbitmq/{{checkcompose_env}}/pki/private/openqa{{checkcompose_env_suffix}}.key" + dest: /etc/pki/fedora-messaging/openqa{{checkcompose_env_suffix}}-key.pem + mode: 0600 + owner: root + group: root + tags: + - config + +- name: Configure fedora-messaging check-compose consumer + template: src=check-compose.toml.j2 dest=/etc/fedora-messaging/check-compose.toml owner=root group=root mode=0640 + notify: + - restart check-compose + tags: + - config + +- name: Wipe the old fedmsg consumer config file + file: path=/etc/fedmsg.d/checkcomp_consumer.py state=absent notify: - restart fedmsg-hub tags: @@ -75,3 +170,6 @@ template: src=check-compose.conf.j2 dest=/etc/check-compose.conf mode=0644 tags: - config + +- name: Enable and start fedora-messaging check-compose consumer + service: name=fm-consumer@check-compose enabled=yes state=started diff --git a/roles/check-compose/templates/check-compose.toml.j2 b/roles/check-compose/templates/check-compose.toml.j2 new file mode 100644 index 0000000000..c75dc7a4ab --- /dev/null +++ b/roles/check-compose/templates/check-compose.toml.j2 @@ -0,0 +1,113 @@ +# fedora-messaging consumer configuration file for relvalconsumer +# (release validation event creator). See relvalconsumer main.yml +# comments for details on the variables that must be set here. +# +# This file is in the TOML format. + +amqp_url = "{{ checkcompose_amqp_url }}" +{% if checkcompose_amqp_passive|bool %} +passive_declares = true +{% endif %} +callback = "check-compose:CheckComposeConsumer" + +[tls] +ca_cert = "{{ checkcompose_amqp_cacert }}" +keyfile = "{{ checkcompose_amqp_key }}" +certfile = "{{ checkcompose_amqp_cert }}" + +[client_properties] +app = "Fedora compose check report sender" +app_url = "https://pagure.io/fedora-qa/check-compose" +app_contacts_email = ["adamwill@fedoraproject.org", "lruzicka@redhat.com", "qa-devel@lists.fedoraproject.org"] + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +[queues."{{ checkcompose_amqp_queue }}"] +durable = false +auto_delete = true +exclusive = true +arguments = {} + +[[bindings]] +queue = "{{ checkcompose_amqp_queue }}" +exchange = "amq.topic" +routing_keys = [{% for key in checkcompose_amqp_routing_keys %}"{{ key }}",{% endfor %}] +# need this to receive messages from ZMQ->AMQP bridge +[[bindings]] +queue = "{{ checkcompose_amqp_queue }}" +exchange = "zmq.topic" +routing_keys = [{% for key in checkcompose_amqp_routing_keys %}"{{ key }}",{% endfor %}] + +[consumer_config] +relval_prod = {{ checkcompose_prod|lower }} + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +{% if checkcompose_amqp_mailto is defined %} +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +formatter = "simple" +level = "ERROR" +mailhost = "{{ checkcompose_amqp_smtp }}" +fromaddr = "{{ checkcompose_amqp_mailfrom }}" +toaddrs = [{% for key in checkcompose_amqp_mailto %}"{{ key }}",{% endfor %}] +subject = "check-compose consumer error" +{% endif %} + +[log_config.loggers.CheckComposeConsumer] +level = "INFO" +propagate = false +{% if checkcompose_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} +handlers = ["console"] +{% endif %} + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +{% if checkcompose_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} +handlers = ["console"] +{% endif %} + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +{% if checkcompose_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} +handlers = ["console"] +{% endif %} + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "ERROR" +{% if checkcompose_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} +handlers = ["console"] +{% endif %} diff --git a/roles/check-compose/templates/checkcomp_consumer.py.j2 b/roles/check-compose/templates/checkcomp_consumer.py.j2 deleted file mode 100644 index d5af64308a..0000000000 --- a/roles/check-compose/templates/checkcomp_consumer.py.j2 +++ /dev/null @@ -1,9 +0,0 @@ -config = { -{% if deployment_type is defined and deployment_type == 'prod' %} - 'checkcomp_consumer.prod.enabled': True, -{% elif deployment_type is defined and deployment_type == 'stg' %} - 'checkcomp_consumer.stg.enabled': True, -{% else %} - checkcomp_consumer.test.enabled': True, -{% endif %} -}