mirror of
https://pagure.io/fedora-infra/ansible.git
synced 2026-04-28 12:32:50 +08:00
ba5f8484a7
This speeds up the process a little bit but not enough for what we want we'll still need to optimize this.
466 lines
16 KiB
Python
466 lines
16 KiB
Python
#!/usr/bin/python -t
|
|
#
|
|
# Create an /etc/gitolite/conf/gitolite.conf file with acls for dist-git
|
|
#
|
|
# Takes no arguments!
|
|
#
|
|
|
|
import copy
|
|
import grp
|
|
import itertools
|
|
import os
|
|
import sys
|
|
import json
|
|
|
|
from multiprocessing import Pool, Manager
|
|
|
|
import requests
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
|
|
|
|
TESTING = False
|
|
|
|
if 'PAGURE_CONFIG' not in os.environ \
|
|
and os.path.exists('/etc/pagure/pagure.cfg'):
|
|
if TESTING:
|
|
print 'Using configuration file `/etc/pagure/pagure.cfg`'
|
|
os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
|
|
|
|
import pagure
|
|
from pagure import SESSION
|
|
from pagure.exceptions import PagureException
|
|
|
|
{% if env == 'staging' %}
|
|
VCS_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/vcs?format=json'
|
|
GRP_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/groups?format=json'
|
|
{% else %}
|
|
VCS_URL = 'https://admin.fedoraproject.org/pkgdb/api/vcs?format=json'
|
|
GRP_URL = 'https://admin.fedoraproject.org/pkgdb/api/groups?format=json'
|
|
{% endif %}
|
|
|
|
|
|
def get_user_info(username):
|
|
''' Uses python-fedora to get information about a FAS user '''
|
|
|
|
user = {
|
|
'username': username,
|
|
'fullname': username,
|
|
'default_email': '%s@fedoraproject.org' % username
|
|
}
|
|
return user
|
|
|
|
|
|
def create_user_obj(session, username):
|
|
''' Creates a sqlalchemy user object for pagure db '''
|
|
try:
|
|
userinfo = get_user_info(username)
|
|
user = pagure.lib.set_up_user(
|
|
session=session,
|
|
username=username,
|
|
fullname=userinfo['fullname'],
|
|
default_email=userinfo['default_email']
|
|
)
|
|
session.commit()
|
|
except SQLAlchemyError:
|
|
session.rollback()
|
|
if TESTING:
|
|
print 'Creating user failed'
|
|
|
|
return user
|
|
|
|
|
|
def create_groups_in_db(groups):
|
|
''' Creates groups in pagure db '''
|
|
|
|
group_keys = groups.keys()
|
|
for groupname in group_keys:
|
|
|
|
# we don't need to do anything with empty groups, do we?
|
|
if len(groups[groupname]) == 0:
|
|
continue
|
|
|
|
# first make sure that the users in the groups are present in the db
|
|
group_users = groups[groupname]
|
|
for guser in group_users:
|
|
user_obj = pagure.lib.search_user(SESSION, username=guser)
|
|
if not user_obj:
|
|
user_obj = create_user_obj(guser)
|
|
|
|
# check if the groups are present in the db
|
|
group_obj = pagure.lib.search_groups(SESSION, group_name=groupname)
|
|
if not group_obj:
|
|
# add the group to the db using the first user in the group
|
|
try:
|
|
pagure.lib.add_group(
|
|
session=SESSION,
|
|
group_name=groupname,
|
|
display_name=groupname,
|
|
description=None,
|
|
group_type='user',
|
|
user=groups[groupname][0],
|
|
is_admin=False,
|
|
blacklist=pagure.APP.config['BLACKLISTED_GROUPS']
|
|
)
|
|
SESSION.commit()
|
|
except SQLAlchemyError:
|
|
SESSION.rollback()
|
|
if TESTING:
|
|
print 'Adding a user to group failed'
|
|
|
|
# now that all groups are present in the db
|
|
# make sure that all the members are there in the group in the db
|
|
for guser in group_users:
|
|
if not pagure.lib.is_group_member(SESSION, guser, groupname):
|
|
group_obj = pagure.lib.search_groups(
|
|
session=SESSION,
|
|
group_name=groupname
|
|
)
|
|
try:
|
|
pagure.lib.add_user_to_group(
|
|
session=SESSION,
|
|
username=guser,
|
|
group=group_obj,
|
|
user=groups[groupname][0],
|
|
is_admin=False
|
|
)
|
|
SESSION.commit()
|
|
except SQLAlchemyError:
|
|
SESSION.rollback()
|
|
if TESTING:
|
|
print 'Adding a user to group failed'
|
|
|
|
|
|
def update_owners_to_db(session, namespace, pkg, owners):
|
|
''' Adds owners to pagure db '''
|
|
pkg_obj = pagure.lib.get_project(
|
|
session, name=pkg, namespace=namespace)
|
|
|
|
for owner in owners:
|
|
# check if the owners are present in the db
|
|
# if not create them
|
|
owner_obj = pagure.lib.search_user(session, username=owner)
|
|
if not owner_obj:
|
|
owner_obj = create_user_obj(session, owner)
|
|
|
|
|
|
# this flag is for avoiding unnecessary db queries
|
|
created = False
|
|
if not pkg_obj:
|
|
try:
|
|
pagure.lib.new_project(
|
|
session=session,
|
|
user=owner,
|
|
namespace=namespace,
|
|
name=pkg,
|
|
blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'],
|
|
allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'],
|
|
gitfolder=pagure.APP.config['GIT_FOLDER'],
|
|
docfolder=pagure.APP.config['DOCS_FOLDER'],
|
|
ticketfolder=pagure.APP.config['TICKETS_FOLDER'],
|
|
requestfolder=pagure.APP.config['REQUESTS_FOLDER'],
|
|
ignore_existing_repo=True,
|
|
)
|
|
session.commit()
|
|
created = True
|
|
except SQLAlchemyError as err:
|
|
session.rollback()
|
|
if TESTING:
|
|
print "Couldn't create project - %s" % pkg
|
|
print "ERROR: %s" % err
|
|
except PagureException as err:
|
|
if TESTING:
|
|
print "Couldn't create project - %s" % pkg
|
|
print "ERROR: %s" % err
|
|
|
|
# so now the pkg surely exists, make the owner,
|
|
# the owner of the repo if he is not
|
|
if created:
|
|
pkg_obj = pagure.lib.get_project(
|
|
session=session,
|
|
name=pkg,
|
|
namespace=namespace
|
|
)
|
|
|
|
#if owner_obj not in pkg_obj.admins and owner_obj is not pkg_obj.user:
|
|
if owner_obj not in pkg_obj.users and owner_obj is not pkg_obj.user:
|
|
try:
|
|
pagure.lib.add_user_to_project(
|
|
session=session,
|
|
project=pkg_obj,
|
|
new_user=owner_obj.user,
|
|
user=pkg_obj.user.user,
|
|
)
|
|
session.commit()
|
|
except SQLAlchemyError as err:
|
|
SESSION.rollback()
|
|
if TESTING:
|
|
print "Couldn't add user to project"
|
|
print "ERROR: %s" % err
|
|
|
|
|
|
def update_groups_to_db(session, namespace, pkg, pkg_groups):
|
|
''' Adds groups to projects in pagure db '''
|
|
|
|
pkg_obj = pagure.lib.get_project(
|
|
session, name=pkg, namespace=namespace)
|
|
|
|
for group in pkg_groups:
|
|
# we have already created all the groups
|
|
group_obj = pagure.lib.search_groups(session, group_name=group)
|
|
|
|
# in case when there are only groups with commit access and no
|
|
# people the flag is for cutting out db queries later
|
|
created = False
|
|
if not pkg_obj:
|
|
try:
|
|
pagure.lib.new_project(
|
|
session=session,
|
|
user=group_obj.creator.user,
|
|
namespace=namespace,
|
|
name=pkg,
|
|
blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'],
|
|
allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'],
|
|
gitfolder=pagure.APP.config['GIT_FOLDER'],
|
|
docfolder=pagure.APP.config['DOCS_FOLDER'],
|
|
ticketfolder=pagure.APP.config['TICKETS_FOLDER'],
|
|
requestfolder=pagure.APP.config['REQUESTS_FOLDER'],
|
|
ignore_existing_repo=True,
|
|
)
|
|
session.commit()
|
|
created = True
|
|
except SQLAlchemyError as err:
|
|
session.rollback()
|
|
if TESTING:
|
|
print "Couldn't create project"
|
|
print "ERROR: %s" % err
|
|
except PagureException as err:
|
|
if TESTING:
|
|
print "Couldn't create project - %s" % pkg
|
|
print "ERROR: %s" % err
|
|
|
|
# for the case when the new project was just created
|
|
# by the above call
|
|
if created:
|
|
pkg_obj = pagure.lib.get_project(
|
|
session, name=pkg, namespace=namespace)
|
|
|
|
# if the group was initially empty, it was not
|
|
# created in the db
|
|
if not group_obj:
|
|
continue
|
|
|
|
# check if the group is added to project
|
|
# if not, add them
|
|
if group_obj not in pkg_obj.groups:
|
|
try:
|
|
pagure.lib.add_group_to_project(
|
|
session=session,
|
|
project=pkg_obj,
|
|
new_group=group,
|
|
user=pkg_obj.user.user,
|
|
access='admin'
|
|
)
|
|
session.commit()
|
|
except SQLAlchemyError as err:
|
|
session.rollback()
|
|
if TESTING:
|
|
print "Adding a group to a project failed"
|
|
print "ERROR: %s" % err
|
|
|
|
|
|
def add_fork_to_gitolite():
|
|
''' Creates a sqlalchemy user object for pagure db '''
|
|
for fork in pagure.lib.search_projects(session=SESSION, fork=True):
|
|
print ''
|
|
print 'repo %s' % (fork.fullname)
|
|
print ' RWC = %s' % fork.user.username
|
|
|
|
|
|
def process_pkg(arg):
|
|
""" Process the given package, adjust pagure for it and queue all the
|
|
ACLs so we can send them to gitolite
|
|
"""
|
|
pkg, acls, myq = arg
|
|
session = pagure.lib.create_session(pagure.APP.config['DB_URL'])
|
|
|
|
branchAcls = {} # Check whether we need to set separate per branch acls
|
|
buffer = [] # Buffer the output per package
|
|
masters = [] # Folks that have commit to master
|
|
writers = [] # Anybody that has write access
|
|
|
|
# Examine each branch in the package
|
|
branches = acls[pkg].keys()
|
|
branches.sort()
|
|
for branch in branches:
|
|
if branch not in ACTIVE.keys():
|
|
continue
|
|
if 'packager' in acls[pkg][branch]['commit']['groups']:
|
|
# If the packager group is defined, everyone has access
|
|
buffer.append(' RWC %s = @all' % (ACTIVE[branch]))
|
|
branchAcls.setdefault('@all', []).append(
|
|
(pkg, ACTIVE[branch])
|
|
)
|
|
if branch == 'master':
|
|
masters.append('@all')
|
|
if '@all' not in writers:
|
|
writers.append('@all')
|
|
else:
|
|
# Extract the owners
|
|
committers = []
|
|
owners = acls[pkg][branch]['commit']['people']
|
|
owners.sort()
|
|
for owner in owners:
|
|
committers.append(owner)
|
|
for group in acls[pkg][branch]['commit']['groups']:
|
|
committers.append('@%s' % group)
|
|
if branch == 'master':
|
|
masters.extend(committers)
|
|
|
|
pkg_groups = acls[pkg][branch]['commit']['groups']
|
|
update_owners_to_db(session, namespace, pkg, owners)
|
|
update_groups_to_db(session, namespace, pkg, pkg_groups)
|
|
|
|
# add all the committers to the top writers list
|
|
for committer in committers:
|
|
if committer not in writers:
|
|
writers.append(committer)
|
|
|
|
# Print the committers to the acl for this package-branch
|
|
committers = ' '.join(committers)
|
|
buffer.append(
|
|
' RWC %s = %s' % (ACTIVE[branch], committers))
|
|
branchAcls.setdefault(committers, []).append(
|
|
(pkg, ACTIVE[branch])
|
|
)
|
|
|
|
session.close()
|
|
data = [pkg, buffer, writers, masters]
|
|
myq.put(data)
|
|
myq.task_done()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
TRUSTED = grp.getgrnam('cvsadmin')[3]
|
|
ARM = grp.getgrnam('fedora-arm')[3]
|
|
SPARC = grp.getgrnam('fedora-sparc')[3]
|
|
IA64 = grp.getgrnam('fedora-ia64')[3]
|
|
S390 = grp.getgrnam('fedora-s390')[3]
|
|
PPC = grp.getgrnam('fedora-ppc')[3]
|
|
PROVEN = grp.getgrnam('provenpackager')[3]
|
|
|
|
# Set the active branches to create ACLs for
|
|
# Give them the git branch eqiv until pkgdb follows suite
|
|
ACTIVE = {
|
|
'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4',
|
|
'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6',
|
|
'epel7': 'epel7',
|
|
'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15':
|
|
'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19',
|
|
'f20': 'f20', 'f21': 'f21', 'f22': 'f22', 'f23': 'f23', 'f24': 'f24',
|
|
'f25': 'f25',
|
|
'devel': 'master', 'master': 'master'}
|
|
|
|
# Create a "regex"ish list 0f the reserved branches
|
|
RESERVED = [
|
|
'f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]',
|
|
'olpc[0-9]']
|
|
|
|
# print out our user groups
|
|
print '@admins = %s' % ' '.join(TRUSTED)
|
|
print '@provenpackager = %s' % ' '.join(PROVEN)
|
|
print '@fedora-arm = %s' % ' '.join(ARM)
|
|
print '@fedora-s390 = %s' % ' '.join(S390)
|
|
print '@fedora-ppc = %s' % ' '.join(PPC)
|
|
|
|
groups = {
|
|
'admins': TRUSTED,
|
|
'fedora-arm': ARM,
|
|
'SPARC': SPARC,
|
|
'IA65': IA64,
|
|
'fedora-s390': S390,
|
|
'fedora-ppc': PPC,
|
|
'provenpackager': PROVEN
|
|
}
|
|
|
|
# Get a list of all the groups
|
|
groups_ = requests.get(GRP_URL).json()
|
|
for group in groups_['groups']:
|
|
print '@%s = %s' % (group, ' '.join(grp.getgrnam(group)[3]))
|
|
gmems = grp.getgrnam(group)[3]
|
|
if group not in groups.keys():
|
|
groups[group] = gmems
|
|
elif groups[group] != gmems:
|
|
groups[group] = gmems
|
|
|
|
create_groups_in_db(groups)
|
|
|
|
# Check the blacklist and if the name clashes
|
|
# append '-1' after them - tmp workaround
|
|
#blacklist = pagure.APP.config.get('BLACKLISTED_PROJECTS')
|
|
#pkgs_list = data['rpms'].keys()
|
|
#for i in pkgs_list:
|
|
#if i in blacklist:
|
|
#data['rpms'][i + '-1'] = data['rpms'].pop(i)
|
|
|
|
data = requests.get(VCS_URL).json()
|
|
|
|
# Give a little space before moving onto the permissions
|
|
print ''
|
|
# print our default permissions
|
|
print 'repo @all'
|
|
print ' - VREF/update-block-push-origin = @all'
|
|
print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc'
|
|
print ' R = @all'
|
|
#print ' RW private- = @all'
|
|
# dont' enable the above until we prevent building for real from private-
|
|
|
|
# XXX - Insert artificial namespaces into the set of namespaces returned
|
|
# by pkgdb. We want to create a mirror of rpms/PKG in test-rpms/PKG
|
|
# This hack occurs in two places. Here, and in the branch-creation script.
|
|
# https://github.com/fedora-infra/pkgdb2/issues/329#issuecomment-207050233
|
|
# And then, this got renamed from rpms-checks to test-rpms
|
|
# https://pagure.io/fedora-infrastructure/issue/5570
|
|
if 'rpms' in data:
|
|
data['test-rpms'] = copy.copy(data['rpms'])
|
|
# Also, modules are a thing
|
|
# https://pagure.io/fedora-infrastructure/issue/5571
|
|
if 'modules' in data:
|
|
data['test-modules'] = copy.copy(data['modules'])
|
|
if 'docker' in data:
|
|
data['test-docker'] = copy.copy(data['docker'])
|
|
|
|
# Get a list of all the packages
|
|
for namespace in data:
|
|
if namespace == 'title':
|
|
continue
|
|
|
|
acls = data[namespace]
|
|
pkglist = sorted(data[namespace].keys())
|
|
|
|
m = Manager()
|
|
q = m.Queue()
|
|
p = Pool(5)
|
|
p.map(process_pkg, itertools.product(pkglist, [acls], [q]))
|
|
p.close()
|
|
p.join()
|
|
|
|
#for pkg in pkglist:
|
|
#process_pkg([pkg, acls, q])
|
|
while q.qsize():
|
|
pkg, buffer, writers, masters = q.get()
|
|
print ''
|
|
print 'repo %s/%s' % (namespace, pkg)
|
|
print '\n'.join(buffer)
|
|
for reserved in RESERVED:
|
|
print ' - %s = @all' % reserved
|
|
print ' RWC refs/tags/ = %s' % ' '.join(writers)
|
|
if masters:
|
|
print ' RWC = %s' % ' '.join(masters)
|
|
|
|
q.join()
|
|
|
|
add_fork_to_gitolite()
|
|
|
|
sys.exit(0)
|