#!/usr/bin/python -tt # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. This script is able to query pkgdb and retrieve for all packages which active branches should be there, browse all the git repos and find out which active branches are missing. It even goes one step further but actually adjusting the git repo by adding the missing branches (or even the missing repo) """ import os import subprocess import requests import fedmsg # Do some off-the-bat configuration of fedmsg. # 1) since this is a one-off script and not a daemon, it needs to connect # to the fedmsg-relay process running on another node (or noone will # hear it) # 2) its going to use the 'shell' certificate which only 'sysadmin' has # read access to. Contrast that with the 'scm' certificate which # everyone in the 'packager' group has access to. config = fedmsg.config.load_config([], None) config['active'] = True config['endpoints']['relay_inbound'] = config['relay_inbound'] fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) {% if env == 'staging' %} PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb' {% else %} PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb' {% endif %} GIT_FOLDER = '/srv/git/rpms/' MKBRANCH = '/usr/local/bin/mkbranch' SETUP_PACKAGE = '/usr/local/bin/setup_git_package' BRANCHES_FROM = { 'epel7': 'f19', 'olpc2': 'f7', 'olpc3': 'f11', 'master': None, } VERBOSE = False class InternalError(Exception): pass class ProcessError(InternalError): pass def _invoke(program, args): '''Run a command and raise an exception if an error occurred. :arg program: The program to invoke :args: List of arguments to pass to the program raises ProcessError if there's a problem. ''' cmdLine = [program] cmdLine.extend(args) if VERBOSE: print ' '.join(cmdLine) if VERBOSE: program = subprocess.Popen( cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) else: program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT) retCode = program.wait() if retCode != 0: e = ProcessError() e.returnCode = retCode e.cmd = ' '.join(cmdLine) if VERBOSE: output = program.stdout.read() e.message = 'Error, "%s" returned %s: %s' % ( e.cmd, e.returnCode, output) print e.message else: e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode) raise e def _create_branch(pkgname, branch): '''Create a specific branch for a package. :arg pkgname: Name of the package to branch :arg branch: Name of the branch to create ''' if branch in BRANCHES_FROM: branch_from = BRANCHES_FROM[branch] else: branch_from = 'master' # Fall back to branching from master. frombranchpath = os.path.join( GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch_from) if not os.path.exists(frombranchpath): branch_from = 'master' branchpath = os.path.join( GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch) if not os.path.exists(branchpath): try: _invoke(MKBRANCH, ['-s', branch_from, branch, pkgname]) except ProcessError, e: if e.returnCode == 255: # This is a warning, not an error return raise finally: fedmsg.publish( topic='branch', modname='git', msg=dict( agent=os.getlogin(), name=pkgname, branch=branch, ), ) elif VERBOSE: print 'Was asked to create branch %s of package %s, but it '\ 'already exists' % (pkgname, branch) def pkgdb_pkg_branch(): """ Queries pkgdb information about VCS and return a dictionnary of which branches are available for which packages. :return: a dict[pkg_name] = [pkg_branches] :rtype: dict """ url = '%s/api/vcs' % PKGDB_URL req = requests.get(url, params={'format': 'json'}) data = req.json() output = {} for pkg in data['packageAcls']: if pkg in output: if VERBOSE: print 'Strange package: %s, it is present twice in the ' \ 'pkgdb output' % pkg output[pkg].updated(data['packageAcls'][pkg].keys()) else: output[pkg] = set(data['packageAcls'][pkg].keys()) return output def get_git_branch(pkg): """ For the specified package name, check the local git and return the list of branches found. """ git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg) if not os.path.exists(git_folder): print 'Could not find %s' % git_folder return set() head_folder = os.path.join(git_folder, 'refs', 'heads') return set(os.listdir(head_folder)) def branch_package(pkgname, branches): '''Create all the branches that are listed in the pkgdb for a package. :arg pkgname: The package to create branches for :arg branches: The branches to creates ''' if VERBOSE: print 'Fixing package %s for branches %s' % (pkgname, branches) # Create the devel branch if necessary if not os.path.exists( os.path.join(GIT_FOLDER, '%s.git' % pkgname)): _invoke(SETUP_PACKAGE, [pkgname]) # Create all the required branches for the package # Use the translated branch name until pkgdb falls inline for branch in branches: _create_branch(pkgname, branch) def main(): """ For each package found via pkgdb, check the local git for its branches and fix inconsistencies. """ local_pkgs = set(os.listdir(GIT_FOLDER)) local_pkgs = set([it.replace('.git', '') for it in local_pkgs]) pkgdb_info = pkgdb_pkg_branch() pkgdb_pkgs = set(pkgdb_info.keys()) ## Commented out as we keep the git of retired packages while they won't ## show up in the information retrieved from pkgdb. #if (local_pkgs - pkgdb_pkgs): #print 'Some packages are present locally but not on pkgdb:' #print ', '.join(sorted(local_pkgs - pkgdb_pkgs)) if (pkgdb_pkgs - local_pkgs): print 'Some packages are present in pkgdb but not locally:' print ', '.join(sorted(pkgdb_pkgs - local_pkgs)) tofix = set() for pkg in sorted(pkgdb_info): pkgdb_branches = pkgdb_info[pkg] git_branches = get_git_branch(pkg) diff = (pkgdb_branches - git_branches) if diff: print '%s missing: %s' % (pkg, ','.join(sorted(diff))) tofix.add(pkg) branch_package(pkg, diff) if tofix: print 'Packages fixed (%s): %s' % ( len(tofix), ', '.join(sorted(tofix))) if __name__ == '__main__': import sys sys.exit(main())