diff --git a/inventory/host_vars/mirrorlist-ibiblio.fedoraproject.org b/inventory/host_vars/mirrorlist-ibiblio.fedoraproject.org index 0ff64f12e2..d9e5bf3439 100644 --- a/inventory/host_vars/mirrorlist-ibiblio.fedoraproject.org +++ b/inventory/host_vars/mirrorlist-ibiblio.fedoraproject.org @@ -2,8 +2,8 @@ nm: 255.255.255.128 gw: 152.19.134.129 dns: 152.2.21.1 -ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-6 -ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL6-x86_64/ +ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-7-ext +ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL7-x86_64/ volgroup: /dev/vg_ibiblio04 eth0_ip: 152.19.134.194 vmhost: ibiblio04.fedoraproject.org diff --git a/inventory/inventory b/inventory/inventory index 053e34a0ea..bda68950b2 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -343,12 +343,10 @@ fedoauth01.stg.phx2.fedoraproject.org memcached01.phx2.fedoraproject.org memcached02.phx2.fedoraproject.org -[mirrorlist] -mirrorlist-ibiblio.fedoraproject.org - [mirrorlist2] mirrorlist-dedicatedsolutions.fedoraproject.org mirrorlist-host1plus.fedoraproject.org +mirrorlist-ibiblio.fedoraproject.org mirrorlist-osuosl.fedoraproject.org mirrorlist-phx2.phx2.fedoraproject.org diff --git a/playbooks/groups/mirrorlist.yml b/playbooks/groups/mirrorlist.yml deleted file mode 100644 index dc039bdeae..0000000000 --- a/playbooks/groups/mirrorlist.yml +++ /dev/null @@ -1,56 +0,0 @@ -# create a new mirrorlist server -# NOTE: should be used with --limit most of the time -# NOTE: make sure there is room/space for this server on the vmhost -# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars - -- name: make mirrorlist app server - hosts: mirrorlist - user: root - gather_facts: False - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - tasks: - - include: "{{ tasks }}/virt_instance_create.yml" - - handlers: - - include: "{{ handlers }}/restart_services.yml" - -- name: make the box be real - hosts: mirrorlist - user: root - gather_facts: True - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - roles: - - base - - rkhunter - - denyhosts - - nagios_client - - geoip - - hosts - - fas_client - - collectd/base - - mirrormanager/mirrorlist - - sudo - - { role: openvpn/client, - when: env != "staging" } - - tasks: - # this is how you include other task lists - - include: "{{ tasks }}/yumrepos.yml" - - include: "{{ tasks }}/2fa_client.yml" - - include: "{{ tasks }}/motd.yml" - - include: "{{ tasks }}/apache.yml" - - include: "{{ tasks }}/mod_wsgi.yml" - - - handlers: - - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/mirrormanager/mirrorlist/files/mirrorlist_server.py b/roles/mirrormanager/mirrorlist/files/mirrorlist_server.py deleted file mode 100755 index 99c7a0c3fe..0000000000 --- a/roles/mirrormanager/mirrorlist/files/mirrorlist_server.py +++ /dev/null @@ -1,1048 +0,0 @@ -#!/usr/bin/python -# -# Copyright (c) 2007-2013 Dell, Inc. -# by Matt Domsch -# Licensed under the MIT/X11 license - -# standard library modules in alphabetical order -import datetime -import getopt -import logging -import logging.handlers -import os -import random -import cPickle as pickle -import select -import signal -import socket -from SocketServer import StreamRequestHandler, ForkingMixIn, UnixStreamServer, BaseServer -import sys -from string import zfill, atoi -import time -import traceback - -try: - import threading -except ImportError: - import dummy_threading as threading - -# not-so-standard library modules that this program needs -from IPy import IP -import GeoIP -import radix -from weighted_shuffle import weighted_shuffle - -# can be overridden on the command line -pidfile = '/var/run/mirrormanager/mirrorlist_server.pid' -socketfile = '/var/run/mirrormanager/mirrorlist_server.sock' -cachefile = '/var/lib/mirrormanager/mirrorlist_cache.pkl' -internet2_netblocks_file = '/var/lib/mirrormanager/i2_netblocks.txt' -global_netblocks_file = '/var/lib/mirrormanager/global_netblocks.txt' -logfile = None -debug = False -must_die = False -# at a point in time when we're no longer serving content for versions -# that don't use yum prioritymethod=fallback -# (e.g. after Fedora 7 is past end-of-life) -# then we can set this value to True -# this only affects results requested using path=... -# for dirs which aren't repositories (such as iso/) -# because we don't know the Version associated with that dir here. -default_ordered_mirrorlist = False - -gipv4 = None -gipv6 = None - -# key is strings in tuple (repo.prefix, arch) -mirrorlist_cache = {} - -# key is directory.name, returns keys for mirrorlist_cache -directory_name_to_mirrorlist = {} - -# key is an IPy.IP structure, value is list of host ids -host_netblock_cache = {} - -# key is hostid, value is list of countries to allow -host_country_allowed_cache = {} - -repo_arch_to_directoryname = {} - -# redirect from a repo with one name to a repo with another -repo_redirect = {} -country_continent_redirect_cache = {} - -# our own private copy of country_continents to be edited -country_continents = GeoIP.country_continents - -disabled_repositories = {} -host_bandwidth_cache = {} -host_country_cache = {} -host_max_connections_cache = {} -file_details_cache = {} -hcurl_cache = {} -asn_host_cache = {} -internet2_tree = radix.Radix() -global_tree = radix.Radix() -host_netblocks_tree = radix.Radix() -netblock_country_tree = radix.Radix() -location_cache = {} -netblock_country_cache = {} - -## Set up our syslog data. -syslogger = logging.getLogger('mirrormanager') -syslogger.setLevel(logging.INFO) -handler = logging.handlers.SysLogHandler(address='/dev/log', facility=logging.handlers.SysLogHandler.LOG_LOCAL4) -syslogger.addHandler(handler) - -def lookup_ip_asn(tree, ip): - """ @t is a radix tree - @ip is an IPy.IP object which may be contained in an entry in l - """ - node = tree.search_best(ip.strNormal()) - if node is None: - return None - return node.data['asn'] - - -def uniqueify(seq, idfun=None): - # order preserving - if idfun is None: - def idfun(x): return x - seen = {} - result = [] - for item in seq: - marker = idfun(item) - # in old Python versions: - # if seen.has_key(marker) - # but in new ones: - if marker in seen: continue - seen[marker] = 1 - result.append(item) - return result - -##### Metalink Support ##### - -def metalink_header(): - # fixme add alternate format pubdate when specified - pubdate = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - doc = '' - doc += '\n' - doc += '\n' - doc += indent(2) + '\n' % (file) - y = detailslist[0] - - def details(y, indentlevel=2): - doc = '' - if y['timestamp'] is not None: - doc += indent(indentlevel+1) + '%s\n' % y['timestamp'] - if y['size'] is not None: - doc += indent(indentlevel+1) + '%s\n' % y['size'] - doc += indent(indentlevel+1) + '\n' - hashes = ('md5', 'sha1', 'sha256', 'sha512') - for h in hashes: - if y[h] is not None: - doc += indent(indentlevel+2) + '%s\n' % (h, y[h]) - doc += indent(indentlevel+1) + '\n' - return doc - - doc += details(y, 2) - # there can be multiple files - if len(detailslist) > 1: - doc += indent(3) + '\n' - for y in detailslist[1:]: - doc += indent(4) + '\n' - doc += details(y,5) - doc += indent(4) + '\n' - doc += indent(3) + '\n' - - doc += indent(3) + '\n' - for (hostid, hcurls) in hosts_and_urls: - private = '' - if hostid not in cache['global']: - private = 'mm0:private="True"' - for url in hcurls: - protocol = url.split(':')[0] - # FIXME January 2010 - # adding protocol= here is not part of the Metalink 3.0 spec, - # but MirrorManager 1.2.6 used it accidentally, as did yum 3.2.20-3 as released - # in Fedora 8, 9, and 10. After those three are EOL (~January 2010), the - # extra protocol= can be removed. - doc += indent(4) + '' % (protocol, protocol, host_country_cache[hostid].upper(), preference, private) - doc += url - doc += '\n' - preference = max(preference-1, 1) - doc += indent(3) + '\n' - doc += indent(2) + '\n' - doc += indent(1) + '\n' - doc += '\n' - return ('metalink', 200, doc) - -def tree_lookup(tree, ip, field, maxResults=None): - # fast lookup in the tree; if present, find all the matching values by deleting the found one and searching again - # this is safe w/o copying the tree again only because this is the only place the tree is used, and - # we'll get a new copy of the tree from our parent the next time it fork()s. - # returns a list of tuples (prefix, data) - result = [] - len_data = 0 - if ip is None: - return result - node = tree.search_best(ip.strNormal()) - while node is not None: - prefix = node.prefix - if type(node.data[field]) == list: - len_data += len(node.data[field]) - else: - len_data += 1 - t = (prefix, node.data[field],) - result.append(t) - if maxResults is None or len_data < maxResults: - tree.delete(prefix) - node = tree.search_best(ip.strNormal()) - else: - break - return result - -def trim_by_client_country(s, clientCountry): - if clientCountry is None: - return s - r = s.copy() - for hostid in s: - if hostid in host_country_allowed_cache and \ - clientCountry not in host_country_allowed_cache[hostid]: - r.remove(hostid) - return r - -def shuffle(s): - l = [] - for hostid in s: - item = (host_bandwidth_cache[hostid], hostid) - l.append(item) - newlist = weighted_shuffle(l) - results = [] - for (bandwidth, hostid) in newlist: - results.append(hostid) - return results - -continents = {} - -def handle_country_continent_redirect(): - new_country_continents = GeoIP.country_continents - for country, continent in country_continent_redirect_cache.iteritems(): - new_country_continents[country] = continent - global country_continents - country_continents = new_country_continents - -def setup_continents(): - new_continents = {} - handle_country_continent_redirect() - for c in country_continents.keys(): - continent = country_continents[c] - if continent not in new_continents: - new_continents[continent] = [c] - else: - new_continents[continent].append(c) - global continents - continents = new_continents - -def do_global(kwargs, cache, clientCountry, header): - c = trim_by_client_country(cache['global'], clientCountry) - header += 'country = global ' - return (header, c) - -def do_countrylist(kwargs, cache, clientCountry, requested_countries, header): - - def collapse(d): - """ collapses a dict {key:set(hostids)} into a set of hostids """ - s = set() - for country, hostids in d.iteritems(): - for hostid in hostids: - s.add(hostid) - return s - - country_cache = {} - for c in requested_countries: - if c in cache['byCountry']: - country_cache[c] = cache['byCountry'][c] - header += 'country = %s ' % c - s = collapse(country_cache) - s = trim_by_client_country(s, clientCountry) - return (header, s) - -def get_same_continent_countries(clientCountry, requested_countries): - result = [] - for r in requested_countries: - if r in country_continents: - requestedCountries = [c.upper() for c in continents[country_continents[r]] \ - if c != clientCountry ] - result.extend(requestedCountries) - uniqueify(result) - return result - -def do_continent(kwargs, cache, clientCountry, requested_countries, header): - if len(requested_countries) > 0: - rc = requested_countries - else: - rc = [clientCountry] - clist = get_same_continent_countries(clientCountry, rc) - return do_countrylist(kwargs, cache, clientCountry, clist, header) - -def do_country(kwargs, cache, clientCountry, requested_countries, header): - if 'GLOBAL' in requested_countries: - return do_global(kwargs, cache, clientCountry, header) - return do_countrylist(kwargs, cache, clientCountry, requested_countries, header) - -def do_netblocks(kwargs, cache, header): - hostresults = set() - if not kwargs.has_key('netblock') or kwargs['netblock'] == "1": - tree_results = tree_lookup(host_netblocks_tree, kwargs['IP'], 'hosts') - for (prefix, hostids) in tree_results: - for hostid in hostids: - if hostid in cache['byHostId']: - hostresults.add((prefix, hostid,)) - header += 'Using preferred netblock ' - return (header, hostresults) - -def do_internet2(kwargs, cache, clientCountry, header): - hostresults = set() - ip = kwargs['IP'] - if ip is None: - return (header, hostresults) - asn = lookup_ip_asn(internet2_tree, ip) - if asn is not None: - header += 'Using Internet2 ' - if clientCountry is not None and clientCountry in cache['byCountryInternet2']: - hostresults = cache['byCountryInternet2'][clientCountry] - hostresults = trim_by_client_country(hostresults, clientCountry) - return (header, hostresults) - -def do_asn(kwargs, cache, header): - hostresults = set() - ip = kwargs['IP'] - if ip is None: - return (header, hostresults) - asn = lookup_ip_asn(global_tree, ip) - if asn is not None and asn in asn_host_cache: - for hostid in asn_host_cache[asn]: - if hostid in cache['byHostId']: - hostresults.add(hostid) - header += 'Using ASN %s ' % asn - return (header, hostresults) - -def do_geoip(kwargs, cache, clientCountry, header): - hostresults = set() - if clientCountry is not None and clientCountry in cache['byCountry']: - hostresults = cache['byCountry'][clientCountry] - header += 'country = %s ' % clientCountry - hostresults = trim_by_client_country(hostresults, clientCountry) - return (header, hostresults) - -def do_location(kwargs, header): - hostresults = set() - if 'location' in kwargs and kwargs['location'] in location_cache: - hostresults = set(location_cache[kwargs['location']]) - header += "Using location %s " % kwargs['location'] - return (header, hostresults) - -def add_host_to_cache(cache, hostid, hcurl): - if hostid not in cache: - cache[hostid] = [hcurl] - else: - cache[hostid].append(hcurl) - return cache - -def append_path(hosts, cache, file, pathIsDirectory=False): - """ given a list of hosts, return a list of objects: - [(hostid, [hcurls]), ... ] - in the same order, appending file if it's not None""" - subpath = None - results = [] - if 'subpath' in cache: - subpath = cache['subpath'] - for hostid in hosts: - hcurls = [] - for hcurl_id in cache['byHostId'][hostid]: - s = hcurl_cache[hcurl_id] - if subpath is not None: - s += "/" + subpath - if file is None and pathIsDirectory: - s += "/" - if file is not None: - if not s.endswith('/'): - s += "/" - s += file - hcurls.append(s) - results.append((hostid, hcurls)) - return results - -def trim_to_preferred_protocols(hosts_and_urls): - """ remove all but http and ftp URLs, - and if both http and ftp are offered, - leave only http. Return [(hostid, url), ...] """ - results = [] - try_protocols = ('https', 'http', 'ftp') - for (hostid, hcurls) in hosts_and_urls: - protocols = {} - url = None - for hcurl in hcurls: - for p in try_protocols: - if hcurl.startswith(p+':'): - protocols[p] = hcurl - - for p in try_protocols: - if p in protocols: - url = protocols[p] - break - - if url is not None: - results.append((hostid, url)) - return results - -def client_ip_to_country(ip): - clientCountry = None - if ip is None: - return None - - # lookup in the cache first - tree_results = tree_lookup(netblock_country_tree, ip, 'country', maxResults=1) - if len(tree_results) > 0: - (prefix, clientCountry) = tree_results[0] - return clientCountry - - # attempt IPv6, then IPv6 6to4 as IPv4, then Teredo, then IPv4 - try: - if ip.version() == 6: - if gipv6 is not None: - clientCountry = gipv6.country_code_by_addr_v6(ip.strNormal()) - if clientCountry is None: - # Try the IPv6-to-IPv4 translation schemes - for scheme in (convert_6to4_v4, convert_teredo_v4): - result = scheme(ip) - if result is not None: - ip = result - break - if ip.version() == 4 and gipv4 is not None: - clientCountry = gipv4.country_code_by_addr(ip.strNormal()) - except: - pass - return clientCountry - -def do_mirrorlist(kwargs): - global debug - global logfile - - def return_error(kwargs, message='', returncode=200): - d = dict(returncode=returncode, message=message, resulttype='mirrorlist', results=[]) - if 'metalink' in kwargs and kwargs['metalink']: - d['resulttype'] = 'metalink' - d['results'] = metalink_failuredoc(message) - return d - - if not (kwargs.has_key('repo') and kwargs.has_key('arch')) and not kwargs.has_key('path'): - return return_error(kwargs, message='# either path=, or repo= and arch= must be specified') - - file = None - cache = None - pathIsDirectory = False - if kwargs.has_key('path'): - path = kwargs['path'].strip('/') - - # Strip duplicate "//" from the path - path = path.replace('//', '/') - - subheader = "# path = %s " % (path) - header = subheader - - sdir = path.split('/') - try: - # path was to a directory - cache = mirrorlist_cache['/'.join(sdir)] - pathIsDirectory=True - except KeyError: - # path was to a file, try its directory - file = sdir[-1] - sdir = sdir[:-1] - try: - cache = mirrorlist_cache['/'.join(sdir)] - except KeyError: - return return_error(kwargs, message=header + 'error: invalid path') - dir = '/'.join(sdir) - else: - if u'source' in kwargs['repo']: - kwargs['arch'] = u'source' - repo = repo_redirect.get(kwargs['repo'], kwargs['repo']) - arch = kwargs['arch'] - subheader = "# repo = %s arch = %s " % (repo, arch) - header = subheader - - if repo in disabled_repositories: - return return_error(kwargs, message=header + 'repo disabled') - try: - dir = repo_arch_to_directoryname[(repo, arch)] - if 'metalink' in kwargs and kwargs['metalink']: - dir += '/repodata' - file = 'repomd.xml' - else: - pathIsDirectory=True - cache = mirrorlist_cache[dir] - except KeyError: - repos = repo_arch_to_directoryname.keys() - repos.sort() - repo_information = header + "error: invalid repo or arch\n" - repo_information += "# following repositories are available:\n" - for i in repos: - if i[0] is not None and i[1] is not None: - repo_information += "# repo=%s&arch=%s\n" % i - return return_error(kwargs, message=repo_information) - - # set kwargs['IP'] exactly once - try: - kwargs['IP'] = IP(kwargs['client_ip']) - except: - kwargs['IP'] = None - - ordered_mirrorlist = cache.get('ordered_mirrorlist', default_ordered_mirrorlist) - done = 0 - location_results = set() - netblock_results = set() - asn_results = set() - internet2_results = set() - country_results = set() - geoip_results = set() - continent_results = set() - global_results = set() - - header, location_results = do_location(kwargs, header) - - requested_countries = [] - if kwargs.has_key('country'): - requested_countries = uniqueify([c.upper() for c in kwargs['country'].split(',') ]) - - # if they specify a country, don't use netblocks or ASN - if not 'country' in kwargs: - header, netblock_results = do_netblocks(kwargs, cache, header) - if len(netblock_results) > 0: - if not ordered_mirrorlist: - done=1 - - if not done: - header, asn_results = do_asn(kwargs, cache, header) - if len(asn_results) + len(netblock_results) >= 3: - if not ordered_mirrorlist: - done = 1 - - clientCountry = client_ip_to_country(kwargs['IP']) - - if clientCountry is None: - print_client_country = "N/A" - else: - print_client_country = clientCountry - - if debug and kwargs.has_key('repo') and kwargs.has_key('arch'): - msg = "IP: %s; DATE: %s; COUNTRY: %s; REPO: %s; ARCH: %s\n" % ( - (kwargs['IP'] or 'None'), time.strftime("%Y-%m-%d"), - print_client_country, kwargs['repo'], kwargs['arch']) - - sys.stdout.write(msg) - sys.stdout.flush() - - if logfile is not None: - logfile.write(msg) - logfile.flush() - - if not done: - header, internet2_results = do_internet2(kwargs, cache, clientCountry, header) - if len(internet2_results) + len(netblock_results) + len(asn_results) >= 3: - if not ordered_mirrorlist: - done = 1 - - if not done and 'country' in kwargs: - header, country_results = do_country(kwargs, cache, clientCountry, requested_countries, header) - if len(country_results) == 0: - header, continent_results = do_continent(kwargs, cache, clientCountry, requested_countries, header) - done = 1 - - if not done: - header, geoip_results = do_geoip(kwargs, cache, clientCountry, header) - if len(geoip_results) >= 3: - if not ordered_mirrorlist: - done = 1 - - if not done: - header, continent_results = do_continent(kwargs, cache, clientCountry, [], header) - if len(geoip_results) + len(continent_results) >= 3: - done = 1 - - if not done: - header, global_results = do_global(kwargs, cache, clientCountry, header) - - def _random_shuffle(s): - l = list(s) - random.shuffle(l) - return l - - def _ordered_netblocks(s): - def ipy_len(t): - (prefix, hostid) = t - return IP(prefix).len() - v4_netblocks = [] - v6_netblocks = [] - for (prefix, hostid) in s: - ip = IP(prefix) - if ip.version() == 4: - v4_netblocks.append((prefix, hostid)) - elif ip.version() == 6: - v6_netblocks.append((prefix, hostid)) - # mix up the order, as sort will preserve same-key ordering - random.shuffle(v4_netblocks) - v4_netblocks.sort(key=ipy_len) - random.shuffle(v6_netblocks) - v6_netblocks.sort(key=ipy_len) - v4_netblocks = [t[1] for t in v4_netblocks] - v6_netblocks = [t[1] for t in v6_netblocks] - return v6_netblocks + v4_netblocks - - def whereismymirror(result_sets): - return_string = 'None' - allhosts = [] - found = False - for (l,s,f) in result_sets: - if len(l) > 0: - allhosts.extend(f(l)) - if not found: - return_string = s - found = True - - allhosts = uniqueify(allhosts) - return allhosts, return_string - - result_sets = [ - (location_results, "location", _random_shuffle), - (netblock_results, "netblocks", _ordered_netblocks), - (asn_results, "asn", _random_shuffle), - (internet2_results, "I2", _random_shuffle), - (country_results, "country", shuffle), - (geoip_results, "geoip", shuffle), - (continent_results, "continent", shuffle), - (global_results, "global", shuffle), - ] - - allhosts, where_string = whereismymirror(result_sets) - try: - ip_str = kwargs['IP'].strNormal() - except: - ip_str = 'Unknown IP' - - pid = str(os.getpid()) - log_string = "mirrorlist(%s): %s found its best mirror from %s (%s)" % (pid, ip_str, where_string, subheader) - syslogger.info(log_string) - - hosts_and_urls = append_path(allhosts, cache, file, pathIsDirectory=pathIsDirectory) - - if 'metalink' in kwargs and kwargs['metalink']: - (resulttype, returncode, results)=metalink(cache, dir, file, hosts_and_urls) - d = dict(message=None, resulttype=resulttype, returncode=returncode, results=results) - return d - - else: - host_url_list = trim_to_preferred_protocols(hosts_and_urls) - d = dict(message=header, resulttype='mirrorlist', returncode=200, results=host_url_list) - return d - -def setup_cache_tree(cache, field): - tree = radix.Radix() - for k, v in cache.iteritems(): - node = tree.add(k.strNormal()) - node.data[field] = v - return tree - -def setup_netblocks(netblocks_file, asns_wanted=None): - tree = radix.Radix() - if netblocks_file is not None: - try: - f = open(netblocks_file, 'r') - except: - return tree - for l in f: - try: - s = l.split() - start, mask = s[0].split('/') - mask = int(mask) - if mask == 0: continue - asn = int(s[1]) - if asns_wanted is None or asn in asns_wanted: - node = tree.add(s[0]) - node.data['asn'] = asn - except: - pass - f.close() - return tree - -def read_caches(): - global mirrorlist_cache - global host_netblock_cache - global host_country_allowed_cache - global host_max_connections_cache - global repo_arch_to_directoryname - global repo_redirect - global country_continent_redirect_cache - global disabled_repositories - global host_bandwidth_cache - global host_country_cache - global file_details_cache - global hcurl_cache - global asn_host_cache - global location_cache - global netblock_country_cache - - data = {} - try: - f = open(cachefile, 'r') - data = pickle.load(f) - f.close() - except: - pass - - if 'mirrorlist_cache' in data: - mirrorlist_cache = data['mirrorlist_cache'] - if 'host_netblock_cache' in data: - host_netblock_cache = data['host_netblock_cache'] - if 'host_country_allowed_cache' in data: - host_country_allowed_cache = data['host_country_allowed_cache'] - if 'repo_arch_to_directoryname' in data: - repo_arch_to_directoryname = data['repo_arch_to_directoryname'] - if 'repo_redirect_cache' in data: - repo_redirect = data['repo_redirect_cache'] - if 'country_continent_redirect_cache' in data: - country_continent_redirect_cache = data['country_continent_redirect_cache'] - if 'disabled_repositories' in data: - disabled_repositories = data['disabled_repositories'] - if 'host_bandwidth_cache' in data: - host_bandwidth_cache = data['host_bandwidth_cache'] - if 'host_country_cache' in data: - host_country_cache = data['host_country_cache'] - if 'file_details_cache' in data: - file_details_cache = data['file_details_cache'] - if 'hcurl_cache' in data: - hcurl_cache = data['hcurl_cache'] - if 'asn_host_cache' in data: - asn_host_cache = data['asn_host_cache'] - if 'location_cache' in data: - location_cache = data['location_cache'] - if 'netblock_country_cache' in data: - netblock_country_cache = data['netblock_country_cache'] - if 'host_max_connections_cache' in data: - host_max_connections_cache = data['host_max_connections_cache'] - - setup_continents() - global internet2_tree - global global_tree - global host_netblocks_tree - global netblock_country_tree - - internet2_tree = setup_netblocks(internet2_netblocks_file) - global_tree = setup_netblocks(global_netblocks_file, asn_host_cache) - # host_netblocks_tree key is a netblock, value is a list of host IDs - host_netblocks_tree = setup_cache_tree(host_netblock_cache, 'hosts') - # netblock_country_tree key is a netblock, value is a single country string - netblock_country_tree = setup_cache_tree(netblock_country_cache, 'country') - -def errordoc(metalink, message): - if metalink: - doc = metalink_failuredoc(message) - else: - doc = message - return doc - -class MirrorlistHandler(StreamRequestHandler): - def handle(self): - signal.signal(signal.SIGHUP, signal.SIG_IGN) - random.seed() - try: - # read size of incoming pickle - readlen = 0 - size = '' - while readlen < 10: - size += self.rfile.read(10 - readlen) - readlen = len(size) - size = atoi(size) - - # read the pickle - readlen = 0 - p = '' - while readlen < size: - p += self.rfile.read(size - readlen) - readlen = len(p) - d = pickle.loads(p) - self.connection.shutdown(socket.SHUT_RD) - except: - pass - - try: - try: - r = do_mirrorlist(d) - except: - raise - message = r['message'] - results = r['results'] - resulttype = r['resulttype'] - returncode = r['returncode'] - except Exception, e: - message=u'# Bad Request %s\n# %s' % (e, d) - exception_msg = traceback.format_exc(e) - sys.stderr.write(message+'\n') - sys.stderr.write(exception_msg) - sys.stderr.flush() - returncode = 400 - results = [] - resulttype = 'mirrorlist' - if d['metalink']: - resulttype = 'metalink' - results = errordoc(d['metalink'], message) - - try: - p = pickle.dumps({'message':message, 'resulttype':resulttype, 'results':results, 'returncode':returncode}) - self.connection.sendall(zfill('%s' % len(p), 10)) - - self.connection.sendall(p) - self.connection.shutdown(socket.SHUT_WR) - except: - pass - -def sighup_handler(signum, frame): - global logfile - if logfile is not None: - name = logfile.name - logfile.close() - logfile = open(name, 'a') - - # put this in a separate thread so it doesn't block clients - if threading.active_count() < 2: - thread = threading.Thread(target=load_databases_and_caches) - thread.daemon = False - try: - thread.start() - except KeyError: - # bug fix for handing an exception when unable to delete from _limbo even though it's not in limbo - # https://code.google.com/p/googleappengine/source/browse/trunk/python/google/appengine/dist27/threading.py?r=327 - pass - -def sigterm_handler(signum, frame): - global must_die - signal.signal(signal.SIGHUP, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - if signum == signal.SIGTERM: - must_die = True - -class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer): - request_queue_size = 300 - def finish_request(self, request, client_address): - signal.signal(signal.SIGHUP, signal.SIG_IGN) - BaseServer.finish_request(self, request, client_address) - -def parse_args(): - global cachefile - global socketfile - global internet2_netblocks_file - global global_netblocks_file - global debug - global logfile - global pidfile - opts, args = getopt.getopt(sys.argv[1:], "c:i:g:p:s:dl:", - ["cache", "internet2_netblocks", "global_netblocks", "pidfile", "socket", "debug", "log="]) - for option, argument in opts: - if option in ("-c", "--cache"): - cachefile = argument - if option in ("-i", "--internet2_netblocks"): - internet2_netblocks_file = argument - if option in ("-g", "--global_netblocks"): - global_netblocks_file = argument - if option in ("-s", "--socket"): - socketfile = argument - if option in ("-p", "--pidfile"): - pidfile = argument - if option in ("-l", "--log"): - try: - logfile = open(argument, 'a') - except: - logfile = None - if option in ("-d", "--debug"): - debug = True - -def open_geoip_databases(): - global gipv4 - global gipv6 - try: - gipv4 = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_STANDARD) - except: - gipv4=None - try: - gipv6 = GeoIP.open("/usr/share/GeoIP/GeoIPv6.dat", GeoIP.GEOIP_STANDARD) - except: - gipv6=None - -def convert_6to4_v4(ip): - all_6to4 = IP('2002::/16') - if ip.version() != 6 or ip not in all_6to4: - return None - parts=ip.strNormal().split(':') - - ab = int(parts[1],16) - a = (ab >> 8) & 0xFF - b = ab & 0xFF - cd = int(parts[2],16) - c = (cd >> 8) & 0xFF - d = cd & 0xFF - - v4addr = '%d.%d.%d.%d' % (a,b,c,d) - return IP(v4addr) - -def convert_teredo_v4(ip): - teredo_std = IP('2001::/32') - teredo_xp = IP('3FFE:831F::/32') - if ip.version() != 6 or (ip not in teredo_std and ip not in teredo_xp): - return None - parts=ip.strNormal().split(':') - - ab = int(parts[6],16) - a = ((ab >> 8) & 0xFF) ^ 0xFF - b = (ab & 0xFF) ^ 0xFF - cd = int(parts[7],16) - c = ((cd >> 8) & 0xFF) ^ 0xFF - d = (cd & 0xFF) ^ 0xFF - - v4addr = '%d.%d.%d.%d' % (a,b,c,d) - return IP(v4addr) - -def load_databases_and_caches(*args, **kwargs): - sys.stderr.write("load_databases_and_caches...") - sys.stderr.flush() - open_geoip_databases() - read_caches() - sys.stderr.write("done.\n") - sys.stderr.flush() - -def remove_pidfile(pidfile): - os.unlink(pidfile) - -def create_pidfile_dir(pidfile): - piddir = os.path.dirname(pidfile) - try: - os.makedirs(piddir, mode=0755) - except OSError, err: - if err.errno == 17: # File exists - pass - else: - raise - except: - raise - -def write_pidfile(pidfile, pid): - create_pidfile_dir(pidfile) - f = open(pidfile, 'w') - f.write(str(pid)+'\n') - f.close() - return 0 - -def manage_pidfile(pidfile): - """returns 1 if another process is running that is named in pidfile, - otherwise creates/writes pidfile and returns 0.""" - pid = os.getpid() - try: - f = open(pidfile, 'r') - except IOError, err: - if err.errno == 2: # No such file or directory - return write_pidfile(pidfile, pid) - return 1 - - oldpid=f.read() - f.close() - - # is the oldpid process still running? - try: - os.kill(int(oldpid), 0) - except ValueError: # malformed oldpid - return write_pidfile(pidfile, pid) - except OSError, err: - if err.errno == 3: # No such process - return write_pidfile(pidfile, pid) - return 1 - - -def main(): - global logfile - global pidfile - signal.signal(signal.SIGHUP, signal.SIG_IGN) - parse_args() - manage_pidfile(pidfile) - - oldumask = os.umask(0) - try: - os.unlink(socketfile) - except: - pass - - load_databases_and_caches() - signal.signal(signal.SIGHUP, sighup_handler) - # restart interrupted syscalls like select - signal.siginterrupt(signal.SIGHUP, False) - ss = ForkingUnixStreamServer(socketfile, MirrorlistHandler) - - while not must_die: - try: - ss.serve_forever() - except select.error: - pass - - try: - os.unlink(socketfile) - except: - pass - - if logfile is not None: - try: - logfile.close() - except: - pass - - remove_pidfile(pidfile) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.exit(-1) diff --git a/roles/mirrormanager/mirrorlist/files/mm-authorized_key b/roles/mirrormanager/mirrorlist/files/mm-authorized_key deleted file mode 100644 index bef6a5561a..0000000000 --- a/roles/mirrormanager/mirrorlist/files/mm-authorized_key +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEA1+Mq0V8RPuCZrjsSz2y56tVR+hykwoYSodhs6ivqkuf16PHo1Aq2PR+I7xnxnJ/fqF20UEV4bmRFs9k4y/QnMOwi90O32vC3WZmfLfoY9S9dzUYXXOYy0lQZPMjmB+yOYZhhGIu2HTc4/CHKjVpCUZKHXwgkouEwsAEHQI86TrepvPwt5GrmAeZjcRp8T3maaFlRf8UFgcajF9ztgiEnBpWBEKaemtUtQ/g9cr/SwNMT3GK+M4qMVaXkuTCeiKwqxueZmTgn76aQ11sfoWYi1lVCyYt02iMDoBLaLERbwBLt3WpY/l3tQaZiPVoRPDH2EQb6v/XISsXNsqGtnc8APfhkVniURNwW/Qz8eXhpnd7GlU90iWPExvGo/Eaj2cNemgiNZH7/U7OKe3/7Li+IpPDhLCfJ8ue7Nqn+2uqhEWvzZXvamzcvEg89PBbgrdHDvJqVLnfsMDuDEo93KZ2pCfTGmjbYjbV3nnArhimLjkkb2E86489F09p65e4AoZw3HFgeW8yA3ecXtXd9FmYYkL0urT1AZFuJ/9B364h4gOqUt0oO7aW0vqhS1hW+7brIJ2DzM/vn90ONk8JVt1T3DvmgmVLxtuo1wu1PkDtekZNtJPGuGzvt5TYDjGPwZzrkYdYazLow3NMCyrhV6oNm8sxuZCLxeyZdM83Q7xty5nM= toshio@puppet1.fedora.phx.redhat.com diff --git a/roles/mirrormanager/mirrorlist/files/mm_sync_data b/roles/mirrormanager/mirrorlist/files/mm_sync_data deleted file mode 100644 index e48bdc7b3b..0000000000 --- a/roles/mirrormanager/mirrorlist/files/mm_sync_data +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -rsync -az --delete-delay --delay-updates --delete mirrormanager@bapp02:/var/lib/mirrormanager/ /var/lib/mirrormanager/ -kill -HUP $(cat /var/run/mirrormanager/mirrorlist_server.pid) diff --git a/roles/mirrormanager/mirrorlist/files/supervisord.conf b/roles/mirrormanager/mirrorlist/files/supervisord.conf deleted file mode 100644 index 2183276e39..0000000000 --- a/roles/mirrormanager/mirrorlist/files/supervisord.conf +++ /dev/null @@ -1,67 +0,0 @@ - -[supervisord] -http_port=/var/tmp/supervisor.sock ; (default is to run a UNIX domain socket server) -;http_port=127.0.0.1:9001 ; (alternately, ip_address:port specifies AF_INET) -;sockchmod=0700 ; AF_UNIX socketmode (AF_INET ignore, default 0700) -;sockchown=nobody.nogroup ; AF_UNIX socket uid.gid owner (AF_INET ignores) -;umask=022 ; (process file creation umask;default 022) -logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) -logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) -logfile_backups=10 ; (num of main logfile rotation backups;default 10) -loglevel=info ; (logging level;default info; others: debug,warn) -pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -nodaemon=false ; (start in foreground if true;default false) -minfds=1024 ; (min. avail startup file descriptors;default 1024) -minprocs=200 ; (min. avail process descriptors;default 200) - -;nocleanup=true ; (don't clean up tempfiles at start;default false) -;http_username=user ; (default is no username (open system)) -;http_password=123 ; (default is no password (open system)) -;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) -;user=chrism ; (default is current user, required if root) -;directory=/tmp ; (default is not to cd during start) -;environment=KEY=value ; (key value pairs to add to environment) - -[supervisorctl] -serverurl=unix:///var/tmp/supervisor.sock ; use a unix:// URL for a unix socket -;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket -;username=chris ; should be same as http_username if set -;password=123 ; should be same as http_password if set -;prompt=mysupervisor ; cmd line prompt (default "supervisor") - -; The below sample program section shows all possible program subsection values, -; create one or more 'real' program: sections to be able to control them under -; supervisor. - -;[program:theprogramname] -;command=/bin/cat ; the program (relative uses PATH, can take args) -;priority=999 ; the relative start priority (default 999) -;autostart=true ; start at supervisord start (default: true) -;autorestart=true ; retstart at unexpected quit (default: true) -;startsecs=10 ; number of secs prog must stay running (def. 10) -;startretries=3 ; max # of serial start failures (default 3) -;exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) -;stopsignal=QUIT ; signal used to kill process (default TERM) -;stopwaitsecs=10 ; max num secs to wait before SIGKILL (default 10) -;user=chrism ; setuid to this UNIX account to run the program -;log_stdout=true ; if true, log program stdout (default true) -;log_stderr=true ; if true, log program stderr (def false) -;logfile=/var/log/cat.log ; child log path, use NONE for none; default AUTO -;logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;logfile_backups=10 ; # of logfile backups (default 10) - -[program:mirrorlist_server] -command=/usr/share/mirrormanager/mirrorlist-server/mirrorlist_server.py -priority=1 -autostart=true -autorestart=true -startsecs=10 -startretries=5 -stopsignal=TERM -stopwaitsecs=10 -user=mirrormanager -log_stderr=true -log_stdout=true -logfile=/var/log/mirrormanager/mirrorlist-server.log -logfile_maxbytes=50MB -logfile_backups=10 diff --git a/roles/mirrormanager/mirrorlist/tasks/main.yml b/roles/mirrormanager/mirrorlist/tasks/main.yml deleted file mode 100644 index ccca26f883..0000000000 --- a/roles/mirrormanager/mirrorlist/tasks/main.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# tasklist for setting up the mirrorlist app components -# create mirrormanager user -- name: add mirrormanager user - uid {{ mirrormanager_uid }} - user: name=mirrormanager uid={{ mirrormanager_uid }} state=present home=/home/mirrormanager createhome=yes - -- name: install supervisor and mirrormanager - yum: pkg={{ item }} state=present - with_items: - - supervisor - - mirrormanager - tags: - - packages - -# Put in the HOTFIX -- name: HOTFIX mirrorlist_server.py - copy: > - src=mirrorlist_server.py dest=/usr/share/mirrormanager/mirrorlist-server/mirrorlist_server.py - owner=root group=root mode=0755 - tags: - - files - notify: - - restart httpd - - restart supervisord - -# mirrormanager user ssh key(s) -- name: add authorized_keys for mirrormanager - authorized_key: key="{{ item }}" user=mirrormanager state=present - with_file: - - mm-authorized_key - -# install mirrorlist-server.conf apache config -- name: mirrorlist-server apache conf - template: src=mirrorlist-server.conf dest=/etc/httpd/conf.d/mirrorlist-server.conf - notify: - - restart apache - tags: - - config - -# nuke mirrormanager.conf so that it doesn't start up -- name: kill /etc/httpd/conf.d/mirrrormanager.conf - copy: 'content="#blanked on purpose" dest=/etc/httpd/conf.d/mirrormanager.conf' - when: inventory_hostname.startswith('mirrorlist') - notify: - - restart apache - tags: - - config - -# selinux policy - mirrormanager - put in place - for the sockfile -# - -# setup and configure supervisord -- name: /etc/supervisord.conf - copy: src=supervisord.conf dest=/etc/supervisord.conf mode=0644 - notify: - - restart supervisord - -- name: enable supervisord - service: name=supervisord state=started enabled=yes - diff --git a/roles/mirrormanager/mirrorlist/templates/mirrorlist-server.conf b/roles/mirrormanager/mirrorlist/templates/mirrorlist-server.conf deleted file mode 100644 index 635c4c9ede..0000000000 --- a/roles/mirrormanager/mirrorlist/templates/mirrorlist-server.conf +++ /dev/null @@ -1,56 +0,0 @@ -KeepAlive off - -Alias /mirrorlists /var/lib/mirrormanager/mirrorlists/ -Alias /publiclist /var/lib/mirrormanager/mirrorlists/publiclist/ -Alias /static /var/lib/mirrormanager/mirrorlists/static/ - - - Options Indexes FollowSymLinks - - -WSGIDaemonProcess mirrorlist user=apache processes={{ mirrorlist_procs }} threads=1 display-name=mirrorlist maximum-requests=1000 - -WSGIScriptAlias /metalink /usr/share/mirrormanager/mirrorlist-server/mirrorlist_client.wsgi -WSGIScriptAlias /mirrorlist /usr/share/mirrormanager/mirrorlist-server/mirrorlist_client.wsgi - -# Set this if you do not have a Reverse Proxy (HTTP Accelerator) that -# is in front of your application server running this code. -# SetEnv mirrorlist_client.noreverseproxy 1 - - - WSGIProcessGroup mirrorlist - - # Apache 2.4 - Require all granted - - - # Apache 2.2 - Order deny,allow - Allow from all - - - - - WSGIProcessGroup mirrorlist - - # Apache 2.4 - Require all granted - - - # Apache 2.2 - Order deny,allow - Allow from all - - - - - - # Apache 2.4 - Require all granted - - - # Apache 2.2 - Order deny,allow - Allow from all - - diff --git a/roles/mirrormanager/mirrorlist/vars/main.yml b/roles/mirrormanager/mirrorlist/vars/main.yml deleted file mode 100644 index 089784c9b0..0000000000 --- a/roles/mirrormanager/mirrorlist/vars/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -mirrormanager_uid: 441 -mirrormanager_gid: 441 -mirrors_gid: 263 -mirrors2_gid: 529