modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-24 10:43:16 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

MBSConsoleHandler: show status of ongoing repository downloads

Browse Source 
When downloading files from Koji to make a local repository, display
a temporary status of which files are being displayed to the console
appended after any log messages. Updates are done by erasing the status
that was written, adding a log message, then writing the status again.
This commit is contained in:
Owen W. Taylor
2020-11-09 13:12:01 -05:00
committed by breilly
parent f5100609aa
commit 646b0590ee
6 changed files with 407 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
module_build_service/builder/utils.py
View File

@@ -83,6 +83,8 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
# Placed here to avoid py2/py3 conflicts...
import koji
log.local_repo_start(tag)
if not archs:
archs = ["x86_64", "noarch"]
@@ -100,6 +102,7 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
if not builds:
log.debug("No builds are associated with the tag %r", tag)
log.local_repo_done(tag, 'No builds to download')
return False
# Reformat builds so they are dict with build_id as a key.
@@ -130,9 +133,12 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
os.remove(local_fn)
repo_changed = True
url = pathinfo.build(build_info) + "/" + fname
download_args.append((url, local_fn))
download_args.append((tag, url, local_fn))
log.info("Downloading %d packages from Koji tag %s to %s" % (len(download_args), tag, repo_dir))
if repo_changed:
log.local_repo_start_downloads(tag, len(download_args), repo_dir)
else:
log.local_repo_done(tag, 'All builds already downloaded')
# Create the output directory
try:
@@ -141,22 +147,26 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
if exception.errno != errno.EEXIST:
raise
def _download_file(url_and_dest):
def _download_file(tag_url_and_dest):
"""
Download a file in a memory efficient manner
:param url_and_dest: a tuple containing the URL and the destination to download to
:param url_and_dest: a tuple containing the tag, the URL and the destination to download to
:return: None
"""
log.info("Downloading {0}...".format(url_and_dest[0]))
if len(url_and_dest) != 2:
raise ValueError("url_and_dest must have two values")
assert len(tag_url_and_dest) == 3, "tag_url_and_dest must have three values"
rv = requests.get(url_and_dest[0], stream=True, timeout=60)
with open(url_and_dest[1], "wb") as f:
tag, url, dest = tag_url_and_dest
log.local_repo_start_download(tag, url)
rv = requests.get(url, stream=True, timeout=60)
with open(dest, "wb") as f:
for chunk in rv.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
log.local_repo_done_download(tag, url)
# Download the RPMs four at a time.
pool = ThreadPool(4)
try:
@@ -173,6 +183,8 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
log.info("Creating local repository in %s" % repo_dir)
execute_cmd(["/usr/bin/createrepo_c", repo_dir])
log.local_repo_done(tag, 'Finished downloading packages')
return True

142
module_build_service/common/logger.py
View File

@@ -24,6 +24,8 @@ import os
import logging
import logging.handlers
import inspect
import re
import signal
import sys
levels = {
@@ -204,6 +206,95 @@ class ModuleBuildLogs(object):
del self.handlers[build.id]
class LocalRepo(object):
def __init__(self, koji_tag):
self.koji_tag = koji_tag
self.current_downloads = set()
self.total_downloads = 0
self.completed_downloads = 0
self.status = ""
def start_downloads(self, total):
self.status = "Downloading packages"
self.total_downloads = total
def start_download(self, url):
self.current_downloads.add(url)
def done_download(self, url):
self.current_downloads.remove(url)
self.completed_downloads += 1
def show_status(self, stream, style):
if self.total_downloads > 0:
count = " {}/{}".format(self.completed_downloads, self.total_downloads)
else:
count = ""
print("{}: {}{}".format(style(self.koji_tag, bold=True), self.status, count),
file=stream)
for url in self.current_downloads:
print(" {}".format(os.path.basename(url)), file=stream)
# Used to split aaa\nbbbb\n to ('aaa', '\n', 'bbb', '\n')
NL_DELIMITER = re.compile('(\n)')
# Matches *common* ANSI control sequences
# https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences
CSI_SEQUENCE = re.compile('\033[0-9;]*[A-Za-z]')
class EraseableStream(object):
"""
Wrapper around a terminal stream for writing output that can be
erased.
"""
def __init__(self, target):
self.target = target
self.lines_written = 0
# We assume that the EraseableStream starts writing at column 0
self.column = 0
self.resize()
def resize(self):
self.size = os.get_terminal_size(self.target.fileno())
def write(self, string):
# We want to track how many lines we've written so that we can
# back up and erase them. Tricky thing is handling wrapping.
# Strip control sequences
plain = CSI_SEQUENCE.sub('', string)
for piece in NL_DELIMITER.split(plain):
if piece == '\n':
self.column = 0
self.lines_written += 1
else:
self.column = self.column + len(piece)
# self.column == self.size.column doesn't wrap -
# normal modern terminals wrap when a character is written
# that would be off-screen, not immediately when the
# line is full.
while self.column > self.size.columns:
self.column -= self.size.columns
self.lines_written += 1
self.target.write(string)
def erase(self):
if self.column > 0:
# move cursor to the beginning of line and delete whole line
self.target.write("\033[0G\033[2K")
for i in range(0, self.lines_written):
# move up cursor and delete whole line
self.target.write("\033[1A\033[2K")
self.lines_written = 0
self.column = 0
FG_COLORS = {
'green': '32',
'red': '91',
@@ -238,8 +329,13 @@ class MBSConsoleHandler(logging.Handler):
self.stream = stream
self.tty = self.stream.isatty()
if self.tty:
self.status_stream = EraseableStream(self.stream)
else:
self.status_stream = None
self.long_running = None
self.repos = {}
self.debug_formatter = logging.Formatter(log_format)
self.info_formatter = logging.Formatter("%(message)s")
@@ -286,6 +382,12 @@ class MBSConsoleHandler(logging.Handler):
self.long_running = None
print(formatted, file=self.stream)
if self.tty:
if self.repos:
print('------------------------------', file=self.status_stream)
for repo in self.repos.values():
repo.show_status(self.status_stream, self.style)
finally:
self.release()
@@ -296,6 +398,10 @@ class MBSConsoleHandler(logging.Handler):
return decorate
def resize(self):
if self.status_stream:
self.status_stream.resize()
@console_message("%s ...")
def long_running_start(self, msg):
if self.long_running:
@@ -315,6 +421,37 @@ class MBSConsoleHandler(logging.Handler):
self.long_running = None
return "{} ... done".format(msg)
@console_message("%s: Making local repository for Koji tag")
def local_repo_start(self, koji_tag):
repo = LocalRepo(koji_tag)
self.repos[koji_tag] = repo
return "{}: Making local repository for Koji tag".format(
self.style(koji_tag, bold=True))
@console_message("%s: %s")
def local_repo_done(self, koji_tag, message):
del self.repos[koji_tag]
return "{}: {}".format(
self.style(koji_tag, bold=True),
self.style(message, fg='green', bold=True))
@console_message("%s: Downloading %d packages from Koji tag to %s")
def local_repo_start_downloads(self, koji_tag, num_packages, repo_dir):
repo = self.repos[koji_tag]
repo.start_downloads(num_packages)
@console_message("%s: Downloading %s")
def local_repo_start_download(self, koji_tag, url):
repo = self.repos[koji_tag]
repo.start_download(url)
@console_message("%s: Done downloading %s")
def local_repo_done_download(self, koji_tag, url):
repo = self.repos[koji_tag]
repo.done_download(url)
@classmethod
def _setup_console_messages(cls):
for value in cls.__dict__.values():
@@ -434,6 +571,11 @@ def init_logging(conf):
root_logger.setLevel(conf.log_level)
handler = MBSConsoleHandler()
root_logger.addHandler(handler)
def handle_sigwinch(*args):
handler.resize()
signal.signal(signal.SIGWINCH, handle_sigwinch)
else:
logging.basicConfig(level=conf.log_level, format=log_format)

4
module_build_service/common/models.py
View File

@@ -606,6 +606,10 @@ class ModuleBuild(MBSBase):
def nvr_string(self):
return kobo.rpmlib.make_nvr(self.nvr)
@property
def nsvc(self):
return "{}:{}:{}:{}".format(self.name, self.stream, self.version, self.context)
@classmethod
def create(
cls,

4
module_build_service/scheduler/batches.py
View File

@@ -255,7 +255,9 @@ def start_next_batch_build(config, module, builder, components=None):
log.info("Skipping build of batch %d, no component to build.", module.batch)
return start_next_batch_build(config, module, builder)
log.info("Starting build of next batch %d, %s" % (module.batch, unbuilt_components))
log.console("Starting build of next batch %d: %s",
module.batch,
', '.join(c.package for c in unbuilt_components))
# Attempt to reuse any components possible in the batch before attempting to build any
unbuilt_components_after_reuse = []

13
module_build_service/scheduler/handlers/modules.py
View File

@@ -145,6 +145,8 @@ def done(msg_id, module_build_id, module_build_state):
build.transition(db_session, conf, state=models.BUILD_STATES["ready"])
db_session.commit()
log.console("Finished building %s", build.nsvc)
build_logs.stop(build)
GenericBuilder.clear_cache(build)
@@ -181,7 +183,8 @@ def init(msg_id, module_build_id, module_build_state):
build_logs.build_logs_dir = mock_resultsdir
build_logs.start(db_session, build)
log.info("Start to handle %s which is in init state.", build.mmd().get_nsvc())
log.console("Starting to build %s", build.nsvc)
log.console("Logging to %s", build_logs.path(db_session, build))
error_msg = ""
failure_reason = "unspec"
@@ -413,7 +416,8 @@ def wait(msg_id, module_build_id, module_build_state):
raise
if not build.component_builds:
log.info("There are no components in module %r, skipping build" % build)
log.console("There are no components in module %s, skipping build",
build.nsvc)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
@@ -426,13 +430,14 @@ def wait(msg_id, module_build_id, module_build_state):
# If all components in module build will be reused, we don't have to build
# module-build-macros, because there won't be any build done.
if attempt_to_reuse_all_components(builder, build):
log.info("All components have been reused for module %r, skipping build" % build)
log.console("All components have been reused for module %s, skipping build",
build.nsvc)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
return []
log.debug("Starting build batch 1")
log.console("Starting build of batch 1: module-build-macros")
build.batch = 1
db_session.commit()