diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py new file mode 100644 index 0000000000..410b64190d --- /dev/null +++ b/files/hotfix/fedimg/consumers.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# This file is part of fedimg. +# Copyright (C) 2014-2017 Red Hat, Inc. +# +# fedimg is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# fedimg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with fedimg; if not, see http://www.gnu.org/licenses, +# or write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: David Gay +# Sayan Chowdhury +""" +This is the `fedmsg consumer`_ that subscribes to the topic emitted after the +completion of the nightly and production compose. The consumer on receving the +message uploads the image using the API of the cloud providers. +""" + +import logging +import multiprocessing.pool + +import fedmsg.consumers +import fedmsg.encoding +import fedfind.release + +import fedimg.uploader + +from fedimg.config import PROCESS_COUNT, STATUS_FILTER +from fedimg.utils import get_rawxz_urls, get_value_from_dict + +LOG = logging.getLogger(__name__) + + +class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): + """ + A `fedmsg consumer`_ that listens to the pungi compose topics and kicks + of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.prod.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.prod.enabled``. + """ + topic = ['org.fedoraproject.prod.pungi.compose.status.change'] + config_key = "fedimgconsumer.prod.enabled" + + def __init__(self, *args, **kwargs): + LOG.info("FedimgConsumer initializing") + super(FedimgConsumer, self).__init__(*args, **kwargs) + + # Threadpool for upload jobs + LOG.info("Creating thread pool of %s process", PROCESS_COUNT) + self.upload_pool = multiprocessing.pool.ThreadPool( + processes=PROCESS_COUNT + ) + LOG.info("FedimgConsumer initialized") + + def consume(self, msg): + """ + This is called when we receive a message matching our topics. + + Args: + msg (dict): The raw message from fedmsg. + """ + LOG.info('Received %r %r', msg['topic'], msg['body']['msg_id']) + + msg_info = msg['body']['msg'] + if msg_info['status'] not in STATUS_FILTER: + return + + location = msg_info['location'] + compose_id = msg_info['compose_id'] + compose_metadata = fedfind.release.get_release(cid=compose_id).metadata + + # Till F27, both cloud-base and atomic images were available + # under variant CloudImages. With F28 and onward releases, + # cloud-base image compose moved to cloud variant and atomic images + # moved under atomic variant. + prev_rel = ['26', '27'] + if msg_info['release_version'] in prev_rel: + images_meta = get_value_from_dict( + compose_metadata, 'images', 'payload', 'images', 'CloudImages', + 'x86_64') + else: + images_meta = get_value_from_dict( + compose_metadata, 'images', 'payload', 'images', + 'Cloud', 'x86_64') + images_meta.extend(get_value_from_dict( + compose_metadata, 'images', 'payload', + 'images', 'AtomicHost', 'x86_64')) + + if images_meta is None: + LOG.debug('No compatible image found to process') + return + + upload_urls = get_rawxz_urls(location, images_meta) + if len(upload_urls) > 0: + LOG.info("Start processing compose id: %s", compose_id) + fedimg.uploader.upload( + pool=self.upload_pool, + urls=upload_urls, + compose_id=compose_id + ) + + +class FedimgStagingConsumer(FedimgConsumer): + """ + A `fedmsg consumer`_ that listens to the staging pungi compose topics and + kicks of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.stg.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.stg.enabled``. + """ + topic = ['org.fedoraproject.stg.pungi.compose.status.change'] + config_key = "fedimgconsumer.stg.enabled" + + +class FedimgDevConsumer(FedimgConsumer): + """ + A `fedmsg consumer`_ that listens to the dev pungi compose topics and + kicks of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.dev.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.dev.enabled``. + """ + topic = ['org.fedoraproject.dev.pungi.compose.status.change'] + config_key = "fedimgconsumer.dev.enabled" + diff --git a/inventory/group_vars/pdc-backend b/inventory/group_vars/pdc-backend index 97720e7ac3..4509bc28b4 100644 --- a/inventory/group_vars/pdc-backend +++ b/inventory/group_vars/pdc-backend @@ -23,18 +23,18 @@ csi_relationship: | fedmsg-hub daemon that loads the pdc-updater consumer plugin. However, the pdc-updater plugin is configured to do different things in each place. - On pdc-updater01, the compose handler is enabled which listens for new pungi + On pdc-backend01, the compose handler is enabled which listens for new pungi composes, and stores them in PDC. Fedora QE uses this data. The consumer has only a single thread enabled to avoid OOMing itself with more than one compose at a time. - On pdc-updater02, the modularity handlers are enabled which listen for MBS - activity, and store that in PDC. pdc-updater02 also hosts the retirement + On pdc-backend02, the modularity handlers are enabled which listen for MBS + activity, and store that in PDC. pdc-backend02 also hosts the retirement handler which listens to dist-git for new dead.package files, and propagates the retirement to PDC (by prematurely EOLing the branch). Multiple threads are enabled so that it can work more efficiently on these smaller tasks. - On pdc-updater03, the dep chain handlers are enabled which listen for koji + On pdc-backend03, the dep chain handlers are enabled which listen for koji messages and store dep chain information in PDC, like what rpms depend on what other rpms at build time, and what containers depend on what rpms, etc.. Multiple threads are enabled so that it can work more efficiently on these diff --git a/inventory/group_vars/pdc-backend-stg b/inventory/group_vars/pdc-backend-stg index 425ca623ef..f07babb410 100644 --- a/inventory/group_vars/pdc-backend-stg +++ b/inventory/group_vars/pdc-backend-stg @@ -23,11 +23,11 @@ csi_relationship: | a fedmsg-hub daemon that loads the pdc-updater consumer plugin. However, the pdc-updater plugin is configured to do different things in each place. - On pdc-updater01, the compose handler is enabled which listens for new pungi + On pdc-backend01, the compose handler is enabled which listens for new pungi composes, and stores them in PDC. Fedora QE uses this data. The consumer has only a single thread enabled to avoid OOMing itself with more than one compose at a time. - On pdc-updater02, the dep chain and modularity handlers are enabled which + On pdc-backend02, the dep chain and modularity handlers are enabled which listen for koji and MBS activity, and store that in PDC. Multiple threads are enabled so that it can work more efficiently on these smaller tasks. diff --git a/inventory/group_vars/resultsdb-prod b/inventory/group_vars/resultsdb-prod index 1df269d999..74833e516b 100644 --- a/inventory/group_vars/resultsdb-prod +++ b/inventory/group_vars/resultsdb-prod @@ -76,3 +76,4 @@ fedmsg_certs: group: apache can_send: - taskotron.result.new + - resultsdb.result.new diff --git a/inventory/group_vars/resultsdb-stg b/inventory/group_vars/resultsdb-stg index d834f9c64a..ec0840caae 100644 --- a/inventory/group_vars/resultsdb-stg +++ b/inventory/group_vars/resultsdb-stg @@ -73,3 +73,4 @@ fedmsg_certs: group: apache can_send: - taskotron.result.new + - resultsdb.result.new diff --git a/playbooks/groups/osbs-cluster.yml b/playbooks/groups/osbs-cluster.yml index 9ea4f06857..0ce6f74fc8 100644 --- a/playbooks/groups/osbs-cluster.yml +++ b/playbooks/groups/osbs-cluster.yml @@ -305,7 +305,8 @@ state: restarted tasks: - + - name: Ensures /etc/dnsmasq.d/ dir exists + file: path="/etc/dnsmasq.d/" state=directory - name: install fedora dnsmasq specific config copy: src: "{{files}}/osbs/fedora-dnsmasq.conf.{{env}}" @@ -324,12 +325,6 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml tasks: - - name: set policy for koji builder in openshift for osbs - shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_stg_username }} && touch /etc/origin/koji-builder-policy-added" - args: - creates: "/etc/origin/koji-builder-policy-added" - when: env == "staging" - - name: set policy for koji builder in openshift for osbs shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_prod_username }} && touch /etc/origin/koji-builder-policy-added" args: @@ -340,6 +335,7 @@ shell: "oadm policy add-role-to-user -n default edit system:serviceaccount:default:builder && touch /etc/origin/atomic-reactor-policy-added" args: creates: "/etc/origin/atomic-reactor-policy-added" + when: env == "production" - name: Deploy OSBS on top of OpenShift hosts: osbs-masters-stg[0]:osbs-masters[0] @@ -399,6 +395,16 @@ tags: - osbs-worker-namespace user: root + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + vars: + osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig + osbs_environment: + KUBECONFIG: "{{ osbs_kubeconfig_path }}" + roles: - role: osbs-namespace osbs_namespace: "{{ osbs_worker_namespace }}" diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index 4b98023843..5ceacbabcb 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -134,3 +134,11 @@ tags: - cron - fedimg + +- name: hotfix - copy the consumers.py over to the site-packages + copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py + notify: + - restart fedmsg-hub + tags: + - fedimg + - hotfix diff --git a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml index ba3ad2a846..652b9005e7 100644 --- a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml +++ b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml @@ -22,6 +22,13 @@ items: maxUnavailable: 25% timeoutSeconds: 600 updatePeriodSeconds: 1 + pre: + failurePolicy: Abort + execNewPod: + containerName: release-monitoring-web + command: [ /bin/sh, -i, -c, "alembic -c /etc/anitya/alembic.ini upgrade head" ] + volumes: + - config-volume type: Rolling template: metadata: diff --git a/roles/openshift-apps/release-monitoring/templates/configmap.yml b/roles/openshift-apps/release-monitoring/templates/configmap.yml index 2392fe8294..aa48fc8cf8 100644 --- a/roles/openshift-apps/release-monitoring/templates/configmap.yml +++ b/roles/openshift-apps/release-monitoring/templates/configmap.yml @@ -70,3 +70,38 @@ data: [anitya_log_config.root] level = "ERROR" handlers = ["console"] + alembic.ini: |- + [alembic] + script_location = anitya:db/migrations + sourceless = false +{% if env == 'staging' %} + sqlalchemy.url = "postgresql://{{ anitya_stg_db_user }}:{{ anitya_stg_db_pass }}@{{ anitya_stg_db_host }}/{{ anitya_stg_db_name }}" +{% else %} + sqlalchemy.url = "postgresql://{{ anitya_db_user }}:{{ anitya_db_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}" +{% endif %} + [loggers] + keys = root,sqlalchemy,alembic + [handlers] + keys = console + [formatters] + keys = generic + [logger_root] + level = WARN + handlers = console + qualname = + [logger_sqlalchemy] + level = WARN + handlers = + qualname = sqlalchemy.engine + [logger_alembic] + level = INFO + handlers = + qualname = alembic + [handler_console] + class = StreamHandler + args = (sys.stderr,) + level = NOTSET + formatter = generic + [formatter_generic] + format = %(levelname)-5.5s [%(name)s] %(message)s + datefmt = %H:%M:%S diff --git a/roles/rkhunter/templates/rkhunter.conf.j2 b/roles/rkhunter/templates/rkhunter.conf.j2 index 9bff3d0c09..28d08530ad 100644 --- a/roles/rkhunter/templates/rkhunter.conf.j2 +++ b/roles/rkhunter/templates/rkhunter.conf.j2 @@ -643,3 +643,7 @@ SCRIPTWHITELIST=/usr/bin/groups SCRIPTWHITELIST=/usr/bin/GET SCRIPTWHITELIST=/sbin/ifup SCRIPTWHITELIST=/sbin/ifdown +{% if inventory_hostname.startswith(('db','pagure','retrace','anitya','upstream')) %} +# Set this size very large on postgres running servers. +IPC_SEG_SIZE=100000000000 +{% endif %}