modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-24 18:52:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

MBSConsoleHandler: show status of ongoing repository downloads

Browse Source 
When downloading files from Koji to make a local repository, display
a temporary status of which files are being displayed to the console
appended after any log messages. Updates are done by erasing the status
that was written, adding a log message, then writing the status again.
This commit is contained in:
Owen W. Taylor
2020-11-09 13:12:01 -05:00
committed by breilly
parent f5100609aa
commit 646b0590ee
6 changed files with 407 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
module_build_service/builder/utils.py
View File

@@ -83,6 +83,8 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
# Placed here to avoid py2/py3 conflicts...
import koji
log.local_repo_start(tag)
if not archs:
archs = ["x86_64", "noarch"]
@@ -100,6 +102,7 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
if not builds:
log.debug("No builds are associated with the tag %r", tag)
log.local_repo_done(tag, 'No builds to download')
return False
# Reformat builds so they are dict with build_id as a key.
@@ -130,9 +133,12 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
os.remove(local_fn)
repo_changed = True
url = pathinfo.build(build_info) + "/" + fname
download_args.append((url, local_fn))
download_args.append((tag, url, local_fn))
log.info("Downloading %d packages from Koji tag %s to %s" % (len(download_args), tag, repo_dir))
if repo_changed:
log.local_repo_start_downloads(tag, len(download_args), repo_dir)
else:
log.local_repo_done(tag, 'All builds already downloaded')
# Create the output directory
try:
@@ -141,22 +147,26 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
if exception.errno != errno.EEXIST:
raise
def _download_file(url_and_dest):
def _download_file(tag_url_and_dest):
"""
Download a file in a memory efficient manner
:param url_and_dest: a tuple containing the URL and the destination to download to
:param url_and_dest: a tuple containing the tag, the URL and the destination to download to
:return: None
"""
log.info("Downloading {0}...".format(url_and_dest[0]))
if len(url_and_dest) != 2:
raise ValueError("url_and_dest must have two values")
assert len(tag_url_and_dest) == 3, "tag_url_and_dest must have three values"
rv = requests.get(url_and_dest[0], stream=True, timeout=60)
with open(url_and_dest[1], "wb") as f:
tag, url, dest = tag_url_and_dest
log.local_repo_start_download(tag, url)
rv = requests.get(url, stream=True, timeout=60)
with open(dest, "wb") as f:
for chunk in rv.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
log.local_repo_done_download(tag, url)
# Download the RPMs four at a time.
pool = ThreadPool(4)
try:
@@ -173,6 +183,8 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
log.info("Creating local repository in %s" % repo_dir)
execute_cmd(["/usr/bin/createrepo_c", repo_dir])
log.local_repo_done(tag, 'Finished downloading packages')
return True