#!/usr/bin/env python from __future__ import print_function import os import sys import openidc_client import argparse import logging import subprocess import requests import koji import time import operator from tabulate import tabulate from multiprocessing.dummy import Pool as ThreadPool from copy import copy import urllib3 import json import requests_kerberos try: from urllib.parse import urljoin except ImportError: from urlparse import urljoin DEFAULT_ID_PROVIDER = "https://id.fedoraproject.org/openidc/" DEFAULT_MBS_SERVER = "https://mbs.fedoraproject.org" DEFAULT_MBS_REST_PREFIX = "/module-build-service/1/" DEFAULT_MBS_REST_API = "{0}module-builds/".format(DEFAULT_MBS_REST_PREFIX) DEFAULT_KOJI_TASK_URL = "https://koji.fedoraproject.org/koji/taskinfo" openidc_client.WEB_PORTS = [13747] BUILD_STATES = { "init": 0, "wait": 1, "build": 2, "done": 3, "failed": 4, "ready": 5, } INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()} def get_auth_method(server, verify=True): config_url = '{0}{1}about/'.format(server.rstrip('/'), DEFAULT_MBS_REST_PREFIX) rv = requests.get(config_url, timeout=30, verify=verify) # Assume that if the connection fails, it's because the config API doesn't # exist on the server yet if not rv.ok: return 'oidc' rv_json = rv.json() return rv_json['auth_method'] def fetch_module_info(server, build_id): if not server: server = DEFAULT_MBS_SERVER idx = int(build_id) response = requests.get(server + '%s/%i?verbose=true' % (DEFAULT_MBS_REST_API, idx)) return response.json() def show_module_info(server, build_id): state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()]) state_names[None] = "undefined" data = fetch_module_info(server, build_id) table = [] for package_name, task_data in data["tasks"].get("rpms", {}).items(): try: koji_task_url = "%s?taskID=%s" % (DEFAULT_KOJI_TASK_URL, task_data['task_id']) except KeyError: koji_task_url = "" table += [[ task_data.get("nvr", "null"), state_names[task_data.get("state", None)], koji_task_url ]] headers = ["NVR", "State", "Koji Task"] print(tabulate(table, headers=headers)) def watch_build(server, build_id): """ Watches the MBS build in a loop, updates every 30 seconds. Returns when build state is 'failed' or 'done' or 'ready' or when user hits ctrl+c. """ done = False while not done: # Clear the screen print(chr(27) + "[2J") state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()]) state_names[None] = "undefined" data = fetch_module_info(server, build_id) tasks = dict() if 'rpms' in data['tasks']: tasks = data['tasks']['rpms'] states = list(set([task['state'] for task in tasks.values()])) inverted = dict() for name, task in tasks.items(): state = task['state'] inverted[state] = inverted.get(state, []) inverted[state].append(name) if 0 in inverted: print("Still building:") for name in inverted[0]: task = tasks[name] print(" ", name, "%s?taskID=%s" % (DEFAULT_KOJI_TASK_URL, task['task_id'])) if 3 in inverted: print("Failed:") for name in inverted[3]: task = tasks[name] print(" ", name, "%s?taskID=%s" % (DEFAULT_KOJI_TASK_URL, task['task_id'])) print() print("Summary:") for state in states: print(" ", len(inverted[state]), "components in the", state_names[state], "state") done = data["state_name"] in ["failed", "done", "ready"] template = ('{owner}\'s build #{id} of {name}-{stream} is in ' 'the "{state_name}" state') if data['state_reason']: template += ' (reason: {state_reason})' if data.get('koji_tag'): template += ' (koji tag: "{koji_tag}")' print(template.format(**data)) if not done: time.sleep(30) # Ideally we would use oidc.send_request here, but it doesn't support # custom HTTP verbs/methods like "PATCH". It sends just "POST"... # TODO: Remove this method once python-openidc-client with verb support # is released and updated in Fedora. def _send_oidc_request(oidc, verb, *args, **kwargs): ckwargs = copy(kwargs) scopes = ckwargs.pop('scopes') new_token = ckwargs.pop('new_token', True) auto_refresh = ckwargs.pop('auto_refresh', True) is_retry = False if oidc.token_to_try: is_retry = True token = oidc.token_to_try oidc.token_to_try = None else: token = oidc.get_token(scopes, new_token=new_token) if not token: return None if oidc.use_post: if 'json' in ckwargs: raise ValueError('Cannot provide json in a post call') if 'data' not in ckwargs: ckwargs['data'] = {} ckwargs['data']['access_token'] = token else: if 'headers' not in ckwargs: ckwargs['headers'] = {} ckwargs['headers']['Authorization'] = 'Bearer %s' % token resp = requests.request(verb, *args, **ckwargs) if resp.status_code == 401 and not is_retry: if not auto_refresh: return resp oidc.token_to_try = oidc.report_token_issue() if not oidc.token_to_try: return resp return _send_oidc_request(oidc, verb, *args, **kwargs) elif resp.status_code == 401: # We got a 401 and this is a retry. Report error oidc.report_token_issue() return resp else: return resp def send_authorized_request(verb, server, url, body, id_provider=None, **kwargs): """ Sends authorized request to server. """ if not server: server = DEFAULT_MBS_SERVER full_url = urljoin(server, url) verify = kwargs.get('verify', True) auth_method = get_auth_method(server, verify=verify) if auth_method == 'oidc': if not id_provider: id_provider = DEFAULT_ID_PROVIDER logging.info("Trying to get the token from %s", id_provider) # Get the auth token using the OpenID client. oidc = openidc_client.OpenIDCClient( "mbs_build", id_provider, {'Token': 'Token', 'Authorization': 'Authorization'}, 'mbs-authorizer', "notsecret") scopes = ['openid', 'https://id.fedoraproject.org/scope/groups', 'https://mbs.fedoraproject.org/oidc/submit-build'] logging.debug("Sending body: %s", body) resp = _send_oidc_request(oidc, verb, full_url, json=body, scopes=scopes, **kwargs) elif auth_method == 'kerberos': if type(body) is dict: data = json.dumps(body) else: data = body auth = requests_kerberos.HTTPKerberosAuth(mutual_authentication=requests_kerberos.OPTIONAL) resp = requests.request(verb, full_url, data=data, auth=auth, verify=verify) if resp.status_code == 401: logging.error('Authentication using Kerberos failed. Make sure you have a valid ' 'Kerberos ticket.') sys.exit(1) else: logging.exception('The MBS server requires an unsupported authentication method of ' '"{0}"'.format(auth_method)) sys.exit(1) return resp def get_scm_url(scm_url, pyrpkg, local=False): """ If `scm_url` it not set, returns the scm_url based on git repository in the `os.getcwd()`. When local is True, file:// scheme is used, otherwise `pyrpkg` is used to determine public URL to git repository. """ if scm_url: return scm_url logging.info("You have not provided SCM URL or branch. Trying to get " "it from current working directory") if local: # Just get the local URL from the current working directory. scm_url = "file://%s" % os.getcwdu() return scm_url else: # Get the url using pyrpkg implementation. process = subprocess.Popen([pyrpkg, 'giturl'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() if process.returncode != 0 and len(err) != 0: logging.error("Cannot get the giturl from current " "working directory using the %s", pyrpkg) logging.error(err) return None scm_url = out[:-1] # remove new-line return scm_url def get_scm_branch(branch=None): """ If `branch` it not set, returns the branch name based on git repository in the `os.getcwd()`. """ if branch: return branch process = subprocess.Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() if process.returncode != 0 and len(err) != 0: logging.error("Cannot get the branch name from current " "working directory.") logging.error(err) return None branch = out[:-1] # remove new-line return branch def submit_module_build(scm_url, branch, server, id_provider, pyrpkg, verify=True, optional=None): """ Submits the module defined by `scm_url` to MBS instance defined by `server`. Returns tuple: build_id or negative error code, error message. """ scm_url = get_scm_url(scm_url, pyrpkg) branch = get_scm_branch(branch) if not scm_url or not branch: return -2, None logging.info("Submitting module build %s", scm_url) body = {'scmurl': scm_url, 'branch': branch} optional = optional if optional else [] try: optional_dict = {y[0]: y[1] for y in [x.split("=", 1) for x in optional]} except IndexError: return -5, "Optional arguments are not in a proper arg=value format." body.update(optional_dict) resp = send_authorized_request( "POST", server, DEFAULT_MBS_REST_API, body, id_provider=id_provider, verify=verify) logging.info(resp.text) data = resp.json() if 'error' in data: return -4, "%s %s: %s" % (data['status'], data['error'], data['message']) elif 'id' in data: return data['id'], None return -3, None def do_local_build(local_builds_nsvs, log_flag=None, yaml_file=None, stream=None, skiptests=False): """ Starts the local build using `mbs-manager build_module_locally`. """ command = ['mbs-manager'] command.append('build_module_locally') if local_builds_nsvs: for build_id in local_builds_nsvs: command += ['--add-local-build', build_id] if not yaml_file: module_dir = os.getcwd() module_name = os.path.basename(module_dir) yaml_file = os.path.join(module_dir, module_name + ".yaml") command.extend(["--file", yaml_file]) if not stream: stream = get_scm_branch() command.extend(["--stream", stream]) if log_flag: command.append(log_flag) if skiptests: command.append("--skiptests") # Some last minute sanity checks before passing off to the other command. if not os.path.exists(yaml_file): logging.error("%s does not exist. Specify --file or check pwd.", yaml_file) return 1 if not stream: logging.error("Unable to determine stream. Either execute " "from a git checkout or pass --stream.") return 1 logging.info("Starting local build of %s, stream %s", yaml_file, stream) process = subprocess.Popen(command) process.communicate() return process.returncode def cancel_module_build(server, id_provider, build_id, verify=True): """ Cancels the module build. """ logging.info("Cancelling module build %s", build_id) resp = send_authorized_request( "PATCH", server, "%s/%s" % (DEFAULT_MBS_REST_API, build_id), {'state': 'failed'}, id_provider=id_provider, verify=verify) logging.info(resp.text) def show_overview(server, finished, limit=30, verify=True): if not server: server = DEFAULT_MBS_SERVER # Base URL to query. baseurl = server + DEFAULT_MBS_REST_API # This logging would break our formatting. logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) def get_module_builds(page=1, state=0): """ Yields modules with state `state`. """ response = requests.get(baseurl, params=dict(page=page, state=state), verify=verify) data = response.json() for item in data['items']: yield item if data['meta']['pages'] > page: for item in get_module_builds(page=page+1, state=state): yield item def get_module_info(module): """ Returns the row with module_info. """ idx = module['id'] response = requests.get(baseurl + '/%i?verbose=true' % idx) module = response.json() n_components = len(module['tasks'].get('rpms', [])) n_built_components = len([c for c in module['tasks'].get('rpms', {}).values() if c['state'] not in [None, 0, koji.BUILD_STATES["BUILDING"]]]) row = [module["id"], module["state_name"], module["time_submitted"], "%s/%s" % (n_built_components, n_components), module["owner"], "%s-%s-%s" % (module["name"], module["stream"], module["version"])] return row if finished: # these are the states when the module build is finished states = [BUILD_STATES["done"], BUILD_STATES["ready"], BUILD_STATES["failed"]] else: # this is when the build is in progress states = [BUILD_STATES["init"], BUILD_STATES["wait"], BUILD_STATES["build"]] # Get all modules in the states we are interested in using 3 threads. pool = ThreadPool(3) module_builds = pool.map(lambda x: list(get_module_builds(state=x)), states) # Make one flat list with all the modules. module_builds = [item for sublist in module_builds for item in sublist] module_builds.sort(key=lambda x: x["id"]) # Get the table rows with information about each module using 20 threads. pool = ThreadPool(20) # get most recent builds table = pool.map(get_module_info, module_builds[-limit:]) # Sort it according to 'id' (first element in list). table = list(reversed(sorted( table, key=operator.itemgetter(0), ))) # Headers for table we will show to user. headers = ["ID", "State", "Submitted", "Components", "Owner", "Module"] print(tabulate(table, headers=headers)) def main(): print('mbs-build is deprecated in favor of fedpkg and rhpkg. Please transition to ' 'using those.', file=sys.stderr) # Parse command line arguments parser = argparse.ArgumentParser(description="Submits and manages module builds.") subparsers = parser.add_subparsers(dest="cmd_name") # logging flag_debug = '-d' flag_verbose = '-v' flag_quiet = '-q' parser.add_argument(flag_debug, dest='debug', action='store_true', help="shows debug output") parser.add_argument(flag_verbose, dest='verbose', action='store_true', help="shows verbose output") parser.add_argument(flag_quiet, dest='quiet', action='store_true', help="shows only errors") parser.add_argument('-k', '--insecure', dest='verify', action='store_false', help="allow connections to SSL sites without certs") parser.add_argument('-s', dest='server', action='store', help="defines the hostname[:port] of the Module Build Service") parser.add_argument('-i', dest='idprovider', action='store', help="defines the OpenID Connect identity provider") parser.add_argument('-p', dest='pyrpkg_client', action='store', help="defines the name of pyrpkg client executable", default="fedpkg") parser_submit = subparsers.add_parser( 'submit', help="submit module build", description="Submits the module build. When 'scm_url' or 'branch' " "is not set, it presumes you are executing this command in " "the directory with the cloned git repository with a module.") parser_submit.add_argument("scm_url", nargs='?') parser_submit.add_argument("branch", nargs='?') parser_submit.add_argument('-o', dest="optional", action='append', help="optional arguments in arg=value format") parser_submit.add_argument('-w', dest="watch", action='store_true', help="watch the build progress") parser_watch = subparsers.add_parser( 'watch', help="watch module build", description="Watches the build progress of a build submitted by " "the 'submit' subcommand.") parser_watch.add_argument("build_id") parser_info = subparsers.add_parser( 'info', help="display detailed information about selected module build", description="Display detailed information about selected module build.") parser_info.add_argument("build_id") parser_cancel = subparsers.add_parser( 'cancel', help="cancel module build", description="Cancels the build submitted by 'submit' subcommand.") parser_cancel.add_argument("build_id") parser_local = subparsers.add_parser( 'local', help="do local build of module", description="Starts local build of a module using the Mock backend. " "When --file or --stream are not set, it presumes you are " "executing this command in the directory with the cloned git " "repository with a modulemd yaml file present.") parser_local.add_argument("--add-local-build", "-l", action='append', dest="local_builds_nsvs", metavar='BUILD_ID') parser_local.add_argument('--file', dest='file', action='store', help="Path to the modulemd yaml file") parser_local.add_argument('--stream', dest='stream', action='store', help=("Name of the stream of this build." " (builds from files only)")) parser_local.add_argument('--skiptests', dest='skiptests', action='store_true', help="Path to the modulemd yaml file") parser_overview = subparsers.add_parser( 'overview', help="show overview of module builds", description="Shows overview of module builds.") parser_overview.add_argument( '--finished', dest='finished', action='store_true', default=False, help="show only finished module builds") parser_overview.add_argument( '--limit', dest='limit', action='store', type=int, default=30, help="the number of recent builds to show") args = parser.parse_args() # Initialize the logging. log_flag = None if args.debug: loglevel = logging.DEBUG log_flag = flag_debug elif args.verbose: loglevel = logging.INFO log_flag = flag_verbose elif args.quiet: loglevel = logging.ERROR log_flag = flag_quiet else: loglevel = logging.WARNING logging.basicConfig(level=loglevel, format="%(levelname)s: %(message)s") if args.verify is False: urllib3.disable_warnings() if args.cmd_name == "submit": # Submit the module build. build_id, errmsg = submit_module_build(args.scm_url, args.branch, args.server, args.idprovider, args.pyrpkg_client, args.verify, args.optional) if build_id < 0: if errmsg: logging.critical(errmsg) sys.exit(build_id) if args.watch: watch_build(args.server, build_id) else: print("Submitted module build %r" % build_id) elif args.cmd_name == "local": sys.exit(do_local_build(args.local_builds_nsvs, log_flag, args.file, args.stream, args.skiptests)) elif args.cmd_name == "watch": # Watch the module build. try: watch_build(args.server, args.build_id) except KeyboardInterrupt: pass elif args.cmd_name == "cancel": # Cancel the module build cancel_module_build(args.server, args.idprovider, args.build_id, args.verify) elif args.cmd_name == "overview": show_overview(args.server, finished=args.finished, limit=args.limit, verify=args.verify) elif args.cmd_name == "info": show_module_info(args.server, args.build_id) if __name__ == "__main__": main()