WarlockFish/fedora-infra_ansible
1
0
Fork 0
mirror of https://pagure.io/fedora-infra/ansible.git synced 2026-03-20 03:57:02 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
419fac6c17e13915ea75744c2095911fa74f3eee
fedora-infra_ansible/roles/bodhi2/backend/files/new-updates-sync
Sinny Kumari 419fac6c17 Necessary changes to accommodate ostree repo move from atomic/ to ostree/ 
Use metalink url for AH ISO creation

Signed-off-by: Sinny Kumari <sinny@redhat.com>
2019-02-07 10:46:40 +01:00

394 lines
17 KiB
Python
Executable File
Raw Blame History

#!/bin/python2
import copy
import fedmsg
import functools
import logging
import subprocess
import os
import sys
import stat
logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger('updates-sync')
SOURCE = '/mnt/koji/compose/updates/'
FEDORADEST = '/pub/fedora/linux/updates/'
FEDORAMODDEST = '/pub/fedora/linux/modular/updates/'
FEDORAALTDEST = '/pub/fedora-secondary/updates/'
EPELDEST = '/pub/epel/'
OSTREESOURCE = '/mnt/koji/compose/ostree/repo/'
OSTREEDEST = '/mnt/koji/ostree/repo/'
RELEASES = {'f29': {'topic': 'fedora',
'version': '29',
'modules': ['fedora', 'fedora-secondary'],
'repos': {'updates': {
'from': 'f29-updates',
'ostrees': [{'ref': 'fedora/29/%(arch)s/updates/atomic-host',
'dest': OSTREEDEST,
'arches': ['x86_64', 'ppc64le', 'aarch64']},
{'ref': 'fedora/29/x86_64/updates/silverblue',
'dest': OSTREEDEST}],
'to': [{'arches': ['x86_64', 'armhfp', 'aarch64', 'source'],
'dest': os.path.join(FEDORADEST, '29', 'Everything')},
{'arches': ['i386', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, '29', 'Everything')}
]},
'updates-testing': {
'from': 'f29-updates-testing',
'ostrees': [{'ref': 'fedora/29/%(arch)s/testing/atomic-host',
'dest': OSTREEDEST,
'arches': ['x86_64', 'ppc64le', 'aarch64']},
{'ref': 'fedora/29/x86_64/testing/silverblue',
'dest': OSTREEDEST}],
'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
'dest': os.path.join(FEDORADEST, 'testing', '29', 'Everything')},
{'arches': ['i386', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, 'testing', '29', 'Everything')}
]}}
},
'f29m': {'topic': 'fedora',
'version': '29m',
'modules': ['fedora', 'fedora-secondary'],
'repos': {'updates': {
'from': 'f29-modular-updates',
'ostrees': [],
'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
'dest': os.path.join(FEDORADEST, '29', 'Modular')},
{'arches': ['i386', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, '29', 'Modular')}
]},
'updates-testing': {
'from': 'f29-modular-updates-testing',
'ostrees': [],
'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
'dest': os.path.join(FEDORADEST, 'testing', '29', 'Modular')},
{'arches': ['i386', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, 'testing', '29', 'Modular')}
]}}
},
'f28': {'topic': 'fedora',
'version': '28',
'modules': ['fedora', 'fedora-secondary'],
'repos': {'updates': {
'from': 'f28-updates',
'ostrees': [{'ref': 'fedora/28/%(arch)s/updates/atomic-host',
'dest': OSTREEDEST,
'arches': ['x86_64', 'ppc64le', 'aarch64']},
{'ref': 'fedora/28/x86_64/updates/workstation',
'dest': OSTREEDEST},
# Hack around for the fact that ostree on f25 doesn't know links
{'ref': 'fedora/28/x86_64/workstation',
'dest': OSTREEDEST}],
'to': [{'arches': ['x86_64', 'armhfp', 'aarch64', 'source'],
'dest': os.path.join(FEDORADEST, '28', 'Everything')},
{'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, '28', 'Everything')}
]},
'updates-testing': {
'from': 'f28-updates-testing',
'ostrees': [{'ref': 'fedora/28/%(arch)s/testing/atomic-host',
'dest': OSTREEDEST,
'arches': ['x86_64', 'ppc64le', 'aarch64']},
{'ref': 'fedora/28/x86_64/testing/workstation',
'dest': OSTREEDEST}],
'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
'dest': os.path.join(FEDORADEST, 'testing', '28', 'Everything')},
{'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, 'testing', '28', 'Everything')}
]}}
},
'f28m': {'topic': 'fedora',
'version': '28m',
'modules': ['fedora', 'fedora-secondary'],
'repos': {'updates': {
'from': 'f28-modular-updates',
'ostrees': [],
'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
'dest': os.path.join(FEDORADEST, '28', 'Modular')},
{'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, '28', 'Modular')}
]},
'updates-testing': {
'from': 'f28-modular-updates-testing',
'ostrees': [],
'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
'dest': os.path.join(FEDORADEST, 'testing', '28', 'Modular')},
{'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
'dest': os.path.join(FEDORAALTDEST, 'testing', '28', 'Modular')}
]}}
},
'epel7': {'topic': 'epel',
'version': '7',
'modules': ['epel'],
'repos': {'epel-testing': {
'from': 'epel7-testing',
'to': [{'arches': ['x86_64', 'aarch64', 'ppc64', 'ppc64le', 'source'],
'dest': os.path.join(EPELDEST, 'testing', '7')}
]},
'epel': {
'from': 'epel7',
'to': [{'arches': ['x86_64', 'aarch64', 'ppc64', 'ppc64le', 'source'],
'dest': os.path.join(EPELDEST, '7')}
]}}
},
'epel6': {'topic': 'epel',
'version': '6',
'modules': ['epel'],
'repos': {'epel': {
'from': 'dist-6E-epel',
'to': [{'arches': ['x86_64', 'ppc64', 'i386', 'source'],
'dest': os.path.join(EPELDEST, '6')}
]},
'epel-testing': {
'from': 'dist-6E-epel-testing',
'to': [{'arches': ['x86_64', 'ppc64', 'i386', 'source'],
'dest': os.path.join(EPELDEST, 'testing', '6')}
]},
}
},
}
# Beneath this is code, no config needed here
FEDMSG_INITED = False
def run_command(cmd):
logger.info('Running %s', cmd)
return subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=False)
def get_ostree_ref(repo, ref):
reffile = os.path.join(repo, 'refs', 'heads', ref)
if not os.path.exists(reffile):
return '----'
with open(reffile, 'r') as f:
return f.read().split()[0]
def sync_ostree(dst, ref):
src_commit = get_ostree_ref(OSTREESOURCE, ref)
dst_commit = get_ostree_ref(dst, ref)
if src_commit == dst_commit:
logger.info('OSTree at %s, ref %s in sync', dst, ref)
else:
print('Syncing ostree ref %s: %s -> %s'
% (ref, src_commit, dst_commit))
logger.info('Syncing OSTree to %s, ref %s: %s -> %s',
dst, ref, src_commit, dst_commit)
cmd = ['ostree', 'pull-local', '--verbose', '--repo',
dst, OSTREESOURCE, ref]
out = run_command(cmd)
cmd = ['ostree', 'summary', '--verbose', '--repo', dst, '--update']
run_command(cmd)
print('Ostree ref %s now at %s' % (ref, src_commit))
def update_fullfilelist(modules):
if not modules:
logger.info('No filelists to update')
return
cmd = ['/usr/local/bin/update-fullfiletimelist', '-l',
'/tmp/update-fullfiletimelist.lock', '-t', '/pub']
cmd.extend(modules)
run_command(cmd)
def rsync(from_path, to_path, excludes=[], link_dest=None, delete=False):
cmd = ['rsync', '-rlptDvHh', '--stats', '--no-human-readable']
for excl in excludes:
cmd += ['--exclude', excl]
if link_dest:
cmd += ['--link-dest', link_dest]
if delete:
cmd += ['--delete', '--delete-delay']
cmd += [from_path, to_path]
stdout = run_command(cmd)
results = {'num_bytes': 0,
'num_deleted': 0}
for line in stdout.split('\n'):
if 'Literal data' in line:
results['num_bytes'] = int(line.split()[2])
elif 'deleting ' in line:
results['num_deleted'] += 1
return results
def collect_stats(stats):
to_collect = ['num_bytes', 'num_deleted']
totals = {}
for stat in to_collect:
totals[stat] = functools.reduce(lambda x, y: x + y,
[r[stat] for r in stats])
return totals
def to_human(num_bytes):
ranges = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']
cur_range = 0
while num_bytes >= 1024 and cur_range+1 <= len(ranges):
num_bytes = num_bytes / 1024
cur_range += 1
return '%s %s' % (num_bytes, ranges[cur_range])
def sync_single_repo_arch(release, repo, arch, dest_path):
source_path = os.path.join(SOURCE,
RELEASES[release]['repos'][repo]['from'],
'compose', 'Everything', arch)
maindir = 'tree' if arch == 'source' else 'os'
results = []
do_drpms = os.path.exists(os.path.join(source_path, maindir, 'drpms'))
results.append(rsync(os.path.join(source_path, maindir, 'Packages'),
os.path.join(dest_path)))
if do_drpms:
results.append(rsync(os.path.join(source_path, maindir, 'drpms'),
os.path.join(dest_path)))
if arch != 'source':
results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'Packages'),
os.path.join(dest_path, 'debug')))
results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'repodata'),
os.path.join(dest_path, 'debug'),
delete=True))
results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'Packages'),
os.path.join(dest_path, 'debug'),
delete=True))
results.append(rsync(os.path.join(source_path, maindir, 'repodata'),
os.path.join(dest_path),
delete=True))
results.append(rsync(os.path.join(source_path, maindir, 'Packages'),
os.path.join(dest_path),
delete=True))
if do_drpms:
results.append(rsync(os.path.join(source_path, maindir, 'drpms'),
os.path.join(dest_path),
delete=True))
return collect_stats(results)
def sync_single_repo(release, repo):
global FEDMSG_INITED
results = []
for archdef in RELEASES[release]['repos'][repo]['to']:
for arch in archdef['arches']:
destarch = 'SRPMS' if arch == 'source' else arch
dest_path = os.path.join(archdef['dest'], destarch)
results.append(sync_single_repo_arch(release, repo, arch, dest_path))
stats = collect_stats(results)
fedmsg_msg = {'repo': repo,
'release': RELEASES[release]['version'],
'bytes': to_human(stats['num_bytes']),
'raw_bytes': str(stats['num_bytes']),
'deleted': str(stats['num_deleted'])}
if not FEDMSG_INITED:
fedmsg.init(active=True, name='relay_inbound', cert_prefix='ftpsync')
FEDMSG_INITED = True
fedmsg.publish(topic='updates.%s.sync' % RELEASES[release]['topic'],
modname='bodhi',
msg=fedmsg_msg)
def determine_last_link(release, repo):
source_path = os.path.join(SOURCE,
RELEASES[release]['repos'][repo]['from'])
target = os.readlink(source_path)
logger.info('Release %s, repo %s, target %s', release, repo, target)
RELEASES[release]['repos'][repo]['from'] = target
return target
def sync_single_release(release):
needssync = False
for repo in RELEASES[release]['repos']:
target = determine_last_link(release, repo)
# if "to" is not empty then sync repo
if RELEASES[release]['repos'][repo]['to']:
curstatefile = os.path.join(
RELEASES[release]['repos'][repo]['to'][0]['dest'], 'state')
curstate = None
if os.path.exists(curstatefile):
with open(curstatefile, 'r') as f:
curstate = f.read().split()[0]
# Resync if Bodhi failed out during the sync waiting, which leads
# to changed repomd.xml without an updated repo.
# (updateinfo is inserted again)
# Fix: https://github.com/fedora-infra/bodhi/pull/1986
if curstate and curstate == target:
curstatestat = os.stat(curstatefile)
repostat = os.stat(os.path.join(
target, 'compose', 'Everything',
RELEASES[release]['repos'][repo]['to'][0]['arches'][0],
'os', 'repodata', 'repomd.xml'))
if curstatestat[stat.ST_MTIME] < repostat[stat.ST_MTIME]:
# If the curstate file has an earlier mtime than the repomd
# of the first architecture, this repo was re-generated
# after the first time it got staged. Resync.
logger.error(
'Re-stage detected of %s %s. '
'State mtime: %s, repo mtime: %s',
release, repo,
curstatestat[stat.ST_MTIME],
repostat[stat.ST_MTIME])
curstate = None
if curstate and curstate == target:
logger.info('This repo has already been synced')
else:
print('Syncing %s %s from %s -> %s' % (release,
repo,
curstate,
target))
sync_single_repo(release, repo)
with open(curstatefile, 'w') as f:
f.write(target)
needssync = True
print('Synced %s %s to %s' % (release, repo, target))
for ostree in RELEASES[release]['repos'][repo].get('ostrees', []):
pairs = []
for arch in ostree.get('arches', ['']):
dest = ostree['dest'] % {'arch': arch}
ref = ostree['ref'] % {'arch': arch}
pairs.append((dest, ref))
for pair in pairs:
sync_ostree(*pair)
return needssync
def main():
to_update = []
for release in sys.argv[1:] or RELEASES:
if sync_single_release(release):
to_update.extend(RELEASES[release]['modules'])
to_update = list(set(to_update))
logger.info('Filelists to update: %s', to_update)
update_fullfilelist(to_update)
if __name__ == '__main__':
main()
Reference in New Issue View Git Blame Copy Permalink