diff --git a/inventory/group_vars/datagrepper b/inventory/group_vars/datagrepper deleted file mode 100644 index 2a37e4c864..0000000000 --- a/inventory/group_vars/datagrepper +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Define resources for this group of hosts here. -# Neeed for rsync from log01 for logs. -custom_rules: ['-A INPUT -p tcp -m tcp -s 10.3.163.39 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT'] -deployment_type: prod -freezes: false -ipa_client_shell_groups: - - fi-apprentice - - sysadmin-datanommer - - sysadmin-noc - - sysadmin-veteran -ipa_client_sudo_groups: - - sysadmin-datanommer -ipa_host_group: datagrepper -ipa_host_group_desc: Service to grep through historical message bus data -lvm_size: 20000 -mem_size: 8192 -num_cpus: 2 -primary_auth_source: ipa -# for systems that do not match the above - specify the same parameter in -# the host_vars/$hostname file -tcp_ports: [80, 443, 6996] diff --git a/inventory/group_vars/datagrepper_stg b/inventory/group_vars/datagrepper_stg deleted file mode 100644 index cc29c5a697..0000000000 --- a/inventory/group_vars/datagrepper_stg +++ /dev/null @@ -1,20 +0,0 @@ ---- -# Define resources for this group of hosts here. -# Neeed for rsync from log01 for logs. -custom_rules: ['-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT'] -freezes: false -ipa_client_shell_groups: - - fi-apprentice - - sysadmin-datanommer - - sysadmin-noc - - sysadmin-veteran -ipa_client_sudo_groups: - - sysadmin-datanommer -ipa_host_group: datagrepper -ipa_host_group_desc: Service to grep through historical message bus data -lvm_size: 20000 -mem_size: 2048 -num_cpus: 1 -# for systems that do not match the above - specify the same parameter in -# the host_vars/$hostname file -tcp_ports: [80, 443, 6996] diff --git a/inventory/inventory b/inventory/inventory index fad2566f20..a105904bbf 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -112,10 +112,10 @@ vmhost-a64-cc01.rdu-cc.fedoraproject.org [datagrepper] #datagrepper01.iad2.fedoraproject.org -datagrepper02.iad2.fedoraproject.org +#datagrepper02.iad2.fedoraproject.org [datagrepper_stg] -datagrepper01.stg.iad2.fedoraproject.org +#datagrepper01.stg.iad2.fedoraproject.org [fedimg] fedimg01.iad2.fedoraproject.org @@ -211,7 +211,6 @@ badges-web01.stg.iad2.fedoraproject.org blockerbugs01.stg.iad2.fedoraproject.org bodhi-backend01.stg.iad2.fedoraproject.org busgateway01.stg.iad2.fedoraproject.org -datagrepper01.stg.iad2.fedoraproject.org koji01.stg.iad2.fedoraproject.org os-node01.stg.iad2.fedoraproject.org os-node02.stg.iad2.fedoraproject.org @@ -637,7 +636,6 @@ copr-db-stg.aws.fedoraproject.org copr-dist-git-dev.aws.fedoraproject.org copr-fe-dev.aws.fedoraproject.org copr-keygen-dev.aws.fedoraproject.org -datagrepper01.stg.iad2.fedoraproject.org db-datanommer01.stg.iad2.fedoraproject.org db-fas01.stg.iad2.fedoraproject.org db-koji01.stg.iad2.fedoraproject.org diff --git a/playbooks/groups/busgateway.yml b/playbooks/groups/busgateway.yml index 03cd1366aa..0d57205974 100644 --- a/playbooks/groups/busgateway.yml +++ b/playbooks/groups/busgateway.yml @@ -39,7 +39,6 @@ roles: - role: fedmsg/hub enable_websocket_server: True - - role: fedmsg/datanommer - role: fedmsg/relay - role: fedmsg/gateway - role: collectd/fedmsg-service diff --git a/playbooks/groups/datagrepper.yml b/playbooks/groups/datagrepper.yml deleted file mode 100644 index 700109d3ea..0000000000 --- a/playbooks/groups/datagrepper.yml +++ /dev/null @@ -1,108 +0,0 @@ -# create a new datagrepper server - -- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=datagrepper:datagrepper_stg" - -- name: make the box be real - hosts: datagrepper:datagrepper_stg - user: root - gather_facts: True - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - roles: - - base - - rkhunter - - nagios_client - - hosts - - { role: openvpn/client, - when: env != "staging" } - - ipa/client - - collectd/base - - fedmsg/base - - { role: rabbit/user, - username: "datagrepper{{ env_suffix }}"} - - rsyncd - - sudo - - mod_wsgi - - pre_tasks: - - import_tasks: "{{ tasks_path }}/yumrepos.yml" - - tasks: - - import_tasks: "{{ tasks_path }}/motd.yml" - - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" - -- name: dole out the service-specific config - hosts: datagrepper:datagrepper_stg - user: root - gather_facts: True - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - roles: - - datagrepper - - role: collectd/web-service - site: datagrepper - url: "http://localhost/datagrepper/raw?delta=86400" - interval: 15 - - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" - -# The gluster work here can be omitted for now. It is used by a feature of -# datagrepper that is partially in place, but not yet functional. -# -#- name: set up gluster server on prod -# hosts: datagrepper:datagrepper_stg -# user: root -# gather_facts: True -# -# vars_files: -# - /srv/web/infra/ansible/vars/global.yml -# - "/srv/private/ansible/vars.yml" -# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml -# -# roles: -# - role: gluster/server -# name: gluster -# username: "{{ datagrepperglusterusername }}" -# password: "{{ datagrepperglusterpassword }}" -# owner: root -# group: root -# datadir: /srv/glusterfs/datagrepper -# -# handlers: -# - import_tasks: "{{ handlers_path }}/restart_services.yml" -# -#- name: set up gluster client on prod -# hosts: datagrepper:datagrepper_stg -# user: root -# gather_facts: True -# -# vars_files: -# - /srv/web/infra/ansible/vars/global.yml -# - "/srv/private/ansible/vars.yml" -# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml -# -# roles: -# - role: gluster/client -# name: gluster -# servers: -# - datagrepper01.iad2.fedoraproject.org -# - datagrepper02.iad2.fedoraproject.org -# username: "{{ datagrepperglusterusername }}" -# password: "{{ datagrepperglusterpassword }}" -# owner: apache -# group: apache -# mountdir: /var/cache/datagrepper -# -# handlers: -# - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/roles/datagrepper/tasks/main.yml b/roles/datagrepper/tasks/main.yml deleted file mode 100644 index e13497076d..0000000000 --- a/roles/datagrepper/tasks/main.yml +++ /dev/null @@ -1,106 +0,0 @@ -- name: install datagrepper and required packages - package: name={{ item }} state=present - with_items: - - datagrepper - - python-psycopg2 - - fedora-messaging - tags: - - packages - - datagrepper - -- name: creating datagrepper config dir - file: path=/etc/datagrepper state=directory owner=root mode=755 - tags: - - config - - datagrepper - -- name: setting up datagrepper config from the template - template: src=datagrepper.cfg dest=/etc/datagrepper/datagrepper.cfg owner=fedmsg group=fedmsg mode=640 - tags: - - config - - datagrepper - -- name: setting up credential from the template - template: src=datagrepper-fedmsg.py dest=/etc/fedmsg.d/datagrepper.py owner=fedmsg group=fedmsg mode=640 - notify: - - reload httpd - tags: - - config - - datagrepper - -- name: remove bum compiled default config file - file: dest="/etc/fedmsg.d/{{item}}" state=absent - with_items: - - datanommer.pyc - - datanommer.pyo - tags: - - config - - datagrepper - -# This file is in the package, empty, and we need to make sure we don't get that one. -- name: empty out default config file - copy: - dest=/etc/fedmsg.d/datanommer.py - content="config={}" - tags: - - config - - datagrepper - -- name: installing memcached package for /topics - package: name=python-memcached state=present - tags: - - packages - - datagrepper - -- name: setting up httpd config - template: src=datagrepper-app.conf dest=/etc/httpd/conf.d/datagrepper.conf owner=root group=root - notify: - - reload httpd - tags: - - config - - datagrepper - -- name: ensure selinux lets httpd talk to the database - seboolean: name=httpd_can_network_connect_db persistent=yes state=yes - tags: - - datagrepper - -- name: ensure selinux lets httpd talk to the memcache - seboolean: name=httpd_can_network_memcache persistent=yes state=yes - tags: - - datagrepper - -# selinux policy has been intentionally omitted since that is obtained from fedmsg/base - -- name: Create /etc/pki/fedora-messaging - file: - dest: /etc/pki/fedora-messaging - mode: 0775 - owner: root - group: root - state: directory - when: "deployment_type is defined" - tags: - - config - -- name: Deploy the Fedora datagrepper fedora-messaging cert - copy: - src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/datagrepper{{env_suffix}}.crt" - dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-cert.pem - mode: 0644 - owner: root - group: root - when: "deployment_type is defined" - tags: - - config - -- name: Deploy the Fedora datagrepper fedora-messaging key - copy: - src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/datagrepper{{env_suffix}}.key" - dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-key.pem - mode: 0640 - owner: root - group: root - when: "deployment_type is defined" - tags: - - config diff --git a/roles/datagrepper/templates/datagrepper-app.conf b/roles/datagrepper/templates/datagrepper-app.conf deleted file mode 100644 index 4bc3782680..0000000000 --- a/roles/datagrepper/templates/datagrepper-app.conf +++ /dev/null @@ -1,42 +0,0 @@ -LoadModule expires_module modules/mod_expires.so -LoadModule headers_module modules/mod_headers.so -LoadModule deflate_module modules/mod_deflate.so - -ExpiresActive On -#ExpiresDefault "access plus 300 seconds" - -ErrorLog logs/datagrepper_error_log -CustomLog logs/datagrepper_access_log combined - -AddOutputFilterByType DEFLATE text/html text/plain text/xml text/javascript text/css application/x-javascript - -# This caching may not necessarily be ideal, or even correct. -# However, it was the only I could get firebug to show me 302's for -# my ToscaWidget resources. - - Header unset Cache-Control - Header unset Etag - Header add Cache-Control "max-age=2592000" - #ExpiresDefault A2592000 - - -# Static resources for the datagrepper app. -Alias /datagrepper/static/ /usr/lib/python2.7/site-packages/datagrepper/static/ - -WSGIDaemonProcess datagrepper user=fedmsg group=fedmsg maximum-requests=50000 display-name=datagrepper processes=20 threads=5 inactivity-timeout=900 -WSGISocketPrefix run/wsgi -WSGIRestrictStdout Off -WSGIRestrictSignal Off -WSGIPythonOptimize 1 - -WSGIScriptAlias /datagrepper /usr/share/datagrepper/apache/datagrepper.wsgi - - - Order deny,allow - Require all granted - - - WSGIProcessGroup datagrepper - Order deny,allow - Require all granted - diff --git a/roles/datagrepper/templates/datagrepper-fedmsg.py b/roles/datagrepper/templates/datagrepper-fedmsg.py deleted file mode 100644 index c54950354a..0000000000 --- a/roles/datagrepper/templates/datagrepper-fedmsg.py +++ /dev/null @@ -1,24 +0,0 @@ -# Configuration for the datagrepper webapp. -config = { - # We don't actually want to run the datanommer consumer on this machine. - 'datanommer.enabled': False, - - # Note that this is connecting to db02. That's fine for now, but we want to - # move the db for datanommer to a whole other db host in the future. We - # expect the amount of data it generates to grow pretty steadily over time - # and we don't want *read* operations on that database to slow down all our - # other apps. - {% if env == "staging" %} - 'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommer_stg_db_password }}@db-datanommer01.stg.iad2.fedoraproject.org/datanommer', - 'fedmenu_url': 'https://apps.stg.fedoraproject.org/fedmenu', - 'fedmenu_data_url': 'https://apps.stg.fedoraproject.org/js/data.js', - {% else %} - 'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01.iad2.fedoraproject.org/datanommer', - 'fedmenu_url': 'https://apps.fedoraproject.org/fedmenu', - 'fedmenu_data_url': 'https://apps.fedoraproject.org/js/data.js', - {% endif %} - - # Only allow ajax/websockets connections back to our domains. - # https://github.com/fedora-infra/datagrepper/pull/192 - 'content_security_policy': 'connect-src https://*.fedoraproject.org wss://*.fedoraproject.org' -} diff --git a/roles/datagrepper/templates/datagrepper.cfg b/roles/datagrepper/templates/datagrepper.cfg deleted file mode 100644 index 1e16528043..0000000000 --- a/roles/datagrepper/templates/datagrepper.cfg +++ /dev/null @@ -1,33 +0,0 @@ -from datetime import timedelta - -### Secret key for the Flask application -SECRET_KEY = '{{ datagrepperCookieSecret }}' - -### Unhappy mako -MAKO_OUTPUT_ENCODING='utf-8' - -{% if env == "staging" %} -DATAGREPPER_BASE_URL='https://apps.stg.fedoraproject.org/datagrepper/' -{% else %} -DATAGREPPER_BASE_URL='https://apps.fedoraproject.org/datagrepper/' -{% endif %} - -DATAGREPPER_CACHE_BACKEND='dogpile.cache.memcached' -DATAGREPPER_CACHE_KWARGS={'arguments': {'url': 'memcached01:11211'}} - -{% if env == "staging" %} -SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db-datanommer01.stg.iad2.fedoraproject.org:5432/datagrepper' -{% else %} -SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db-datanommer01.iad2.fedoraproject.org:5432/datagrepper' -{% endif %} - -{% if env == "staging" %} -DATAGREPPER_OPENID_ENDPOINT='id.stg.fedoraproject.org' -{% else %} -DATAGREPPER_OPENID_ENDPOINT='id.fedoraproject.org' -{% endif %} - -RUNNER_LOCKFILE='/var/run/fedmsg/datagrepper.lock' -JOB_OUTPUT_DIR='/var/cache/datagrepper' -JOB_OUTPUT_URL='https://apps.{% if env == "staging" %}stg.{% endif %}fedoraproject.org/datagrepper/output' -JOB_EXPIRY=timedelta(days=7) diff --git a/roles/fedmsg/datanommer/tasks/main.yml b/roles/fedmsg/datanommer/tasks/main.yml deleted file mode 100644 index ad375a6915..0000000000 --- a/roles/fedmsg/datanommer/tasks/main.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This is a *very* simple role. The config needed for fedmsg-relay to operate -# correctly is actually included as part of the fedmsg/base role. - -- name: install datanommer packages - package: - state: present - name: - - python-datanommer-consumer - - python-datanommer-models - - python-psycopg2 - - datanommer-commands - tags: - - packages - - datanommer - -- name: copy alembic config - template: > - src=alembic.ini - dest=/usr/share/datanommer.models/alembic.ini - mode=0640 - tags: - - config - - datanommer - notify: - - restart fedmsg-hub - -- name: copy datanommer config itself - template: > - src="{{ item }}" - dest="/etc/fedmsg.d/{{item}}" - mode=0640 - owner=fedmsg - group=nrpe - with_items: - - datanommer.py - - fas-credentials.py - tags: - - config - - datanommer - notify: - - restart fedmsg-hub diff --git a/roles/fedmsg/datanommer/templates/alembic.ini b/roles/fedmsg/datanommer/templates/alembic.ini deleted file mode 100644 index fa2b2da55d..0000000000 --- a/roles/fedmsg/datanommer/templates/alembic.ini +++ /dev/null @@ -1,52 +0,0 @@ -[alembic] -# path to migration scripts -script_location = /usr/share/datanommer.models/alembic/ - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -{% if env == "production" %} -sqlalchemy.url = postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01/datanommer -{% else %} -sqlalchemy.url = postgresql://{{ datanommerDBUser }}:{{ datanommer_stg_db_password }}@db-datanommer01.stg.iad2.fedoraproject.org/datanommer -{% endif %} -fedmsg_config_dir = /etc/fedmsg.d/ - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/roles/fedmsg/datanommer/templates/datanommer.py b/roles/fedmsg/datanommer/templates/datanommer.py deleted file mode 100644 index 35179ec395..0000000000 --- a/roles/fedmsg/datanommer/templates/datanommer.py +++ /dev/null @@ -1,9 +0,0 @@ -# Configuration for the datanommer consumer. A plugin for the fedmsg-hub process. -config = { - 'datanommer.enabled': True, -{% if env == "production" %} - 'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01.iad2.fedoraproject.org/datanommer', -{% else %} - 'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommer_stg_db_password }}@db-datanommer01.stg.iad2.fedoraproject.org/datanommer', -{% endif %} -} diff --git a/roles/fedmsg/datanommer/templates/fas-credentials.py b/roles/fedmsg/datanommer/templates/fas-credentials.py deleted file mode 100644 index 76416d2178..0000000000 --- a/roles/fedmsg/datanommer/templates/fas-credentials.py +++ /dev/null @@ -1,9 +0,0 @@ -config = dict( - fas_credentials=dict( - username="fedoradummy", - password="{{ fedoraDummyUserPassword }}", - {% if env == 'staging' %} - base_url="https://accounts.stg.fedoraproject.org/", - {% endif %} - ), -)