diff --git a/contrib/mbs-build b/contrib/mbs-build index 3b8f677c..6b8aed35 100755 --- a/contrib/mbs-build +++ b/contrib/mbs-build @@ -9,6 +9,9 @@ import subprocess import requests import koji import time +import operator +from tabulate import tabulate +from multiprocessing.dummy import Pool as ThreadPool from copy import copy DEFAULT_ID_PROVIDER = "https://id.fedoraproject.org/openidc/" @@ -16,6 +19,17 @@ DEFAULT_MBS_SERVER = "https://mbs.fedoraproject.org" openidc_client.WEB_PORTS = [13747] +BUILD_STATES = { + "init": 0, + "wait": 1, + "build": 2, + "done": 3, + "failed": 4, + "ready": 5, +} + +INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()} + def watch_build(server, build_id): """ Watches the MBS build in a loop, updates every 30 seconds. @@ -240,6 +254,68 @@ def cancel_module_build(server, id_provider, build_id): {'state': 'failed'}) logging.info(resp.text) +def show_overview(server): + if not server: + server = DEFAULT_MBS_SERVER + + # Base URL to query. + baseurl = server + '/module-build-service/1/module-builds/' + + # This logging would break our formatting. + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + def get_module_builds(page=1, state=0): + """ + Yields modules with state `state`. + """ + response = requests.get(baseurl, params=dict(page=page, state=state)) + data = response.json() + for item in data['items']: + yield item + if data['meta']['pages'] > page: + for item in get_module_builds(page=page+1, state=state): + yield item + + def get_module_info(module): + """ + Returns the row with module_info. + """ + idx = module['id'] + response = requests.get(baseurl + '/%i?verbose=true' % idx) + module = response.json() + n_components = len(module['tasks'].get('rpms', [])) + n_built_components = len([c for c in module['tasks'].get('rpms', {}).values() if c['state'] not in [None, 0, koji.BUILD_STATES["BUILDING"]]]) + row = [module["id"], module["state_name"], module["time_submitted"], + "%s/%s" % (n_built_components, n_components), module["owner"], + "%s-%s-%s" % (module["name"], module["stream"], module["version"])] + return row + + # We are interested only in init, wait and build states. + states = [BUILD_STATES["init"], BUILD_STATES["wait"], + BUILD_STATES["build"]] + + # Get all modules in the states we are interested in using 3 threads. + pool = ThreadPool(3) + module_builds = pool.map(lambda x: list(get_module_builds(state=x)), + states) + # Make one flat list with all the modules. + module_builds = [item for sublist in module_builds for item in sublist] + + # Get the table rows with information about each module using 20 threads. + pool = ThreadPool(20) + table = pool.map(get_module_info, module_builds) + + # Sort it according to 'id' (first element in list). + table = list(reversed(sorted( + table, key=operator.itemgetter(0), + ))) + + # Headers for table we will show to user. + headers = ["ID", "State", "Submitted", "Components", "Owner", "Module"] + + print(tabulate(table, headers=headers)) + def main(): # Parse command line arguments parser = argparse.ArgumentParser(description="Submits and manages module builds.") @@ -276,6 +352,7 @@ def main(): 'cancel', help="cancel module build", description="Cancels the build submitted by 'submit' subcommand.") parser_cancel.add_argument("build_id") + parser_local = subparsers.add_parser( 'local', help="do local build of module", description="Starts local build of a module using the Mock backend. " @@ -285,6 +362,10 @@ def main(): parser_local.add_argument("scm_url", nargs='?') parser_local.add_argument("branch", nargs='?') + parser_overview = subparsers.add_parser( + 'overview', help="show overview of module builds", + description="Shows overview of module builds.") + args = parser.parse_args() # Initialize the logging. @@ -316,6 +397,8 @@ def main(): elif args.cmd_name == "cancel": # Cancel the module build cancel_module_build(args.server, args.idprovider, args.build_id) + elif args.cmd_name == "overview": + show_overview(args.server) if __name__ == "__main__": main() diff --git a/requirements.txt b/requirements.txt index 5281b57b..81b8fa1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ qpid-python six sqlalchemy futures # Python 2 only +tabulate