fm-orchestrator/contrib/mbs-build

#!/usr/bin/env python
from __future__ import print_function
import os
import sys
import openidc_client
import argparse
import logging
import subprocess
import requests
import koji
import time
import operator
from tabulate import tabulate
from multiprocessing.dummy import Pool as ThreadPool
from copy import copy

try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

DEFAULT_ID_PROVIDER = "https://id.fedoraproject.org/openidc/"
DEFAULT_MBS_SERVER = "https://mbs.fedoraproject.org"

openidc_client.WEB_PORTS = [13747]

BUILD_STATES = {
    "init": 0,
    "wait": 1,
    "build": 2,
    "done": 3,
    "failed": 4,
    "ready": 5,
}

INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()}


def fetch_module_info(server, build_id):
    if not server:
        server = DEFAULT_MBS_SERVER
    idx = int(build_id)

    response = requests.get(server + '/module-build-service/1/module-builds/%i?verbose=true' % idx)
    return response.json()


def show_module_info(server, build_id):
    state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()])
    state_names[None] = "undefined"

    data = fetch_module_info(server, build_id)
    table = []
    for package_name, task_data in data["tasks"]["rpms"].items():
        try:
            koji_task_url = "https://koji.fedoraproject.org/koji/taskinfo?taskID=%s" % task_data['task_id']
        except KeyError:
            koji_task_url = ""
        table += [[
            task_data.get("nvr", "null"),
            state_names[task_data.get("state", None)],
            koji_task_url
        ]]
    headers = ["NVR", "State", "Koji Task"]

    print(tabulate(table, headers=headers))


def watch_build(server, build_id):
    """
    Watches the MBS build in a loop, updates every 30 seconds.
    Returns when build state is 'failed' or 'done' or 'ready' or when
    user hits ctrl+c.
    """
    done = False
    while not done:
        # Clear the screen
        print(chr(27) + "[2J")

        state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()])
        state_names[None] = "undefined"

        data = fetch_module_info(server, build_id)

        tasks = data['tasks']['rpms']
        states = list(set([task['state'] for task in tasks.values()]))
        inverted = dict()
        for name, task in tasks.items():
            state = task['state']
            inverted[state] = inverted.get(state, [])
            inverted[state].append(name)

        if 0 in inverted:
            print("Still building:")
            for name in inverted[0]:
                task = tasks[name]
                print("  ", name, "https://koji.fedoraproject.org/koji/taskinfo?taskID=%s" % task['task_id'])

        if 3 in inverted:
            print("Failed:")
            for name in inverted[3]:
                task = tasks[name]
                print("  ", name, "https://koji.fedoraproject.org/koji/taskinfo?taskID=%s" % task['task_id'])

        print()
        print("Summary:")
        for state in states:
            print("  ", len(inverted[state]), "components in the", state_names[state], "state")

        done = data["state_name"] in ["failed", "done", "ready"]

        template = ('{owner}\'s build #{id} of {name}-{stream} is in '
                    'the "{state_name}" state')
        if data['state_reason']:
            template += ' (reason: {state_reason})'
        print(template.format(**data))
        time.sleep(30)

# Ideally we would use oidc.send_request here, but it doesn't support
# custom HTTP verbs/methods like "PATCH". It sends just "POST"...
# TODO: Remove this method once python-openidc-client with verb support
# is released and updated in Fedora.
def _send_oidc_request(oidc, verb, *args, **kwargs):
    ckwargs = copy(kwargs)

    scopes = ckwargs.pop('scopes')
    new_token = ckwargs.pop('new_token', True)
    auto_refresh = ckwargs.pop('auto_refresh', True)

    is_retry = False
    if oidc.token_to_try:
        is_retry = True
        token = oidc.token_to_try
        oidc.token_to_try = None
    else:
        token = oidc.get_token(scopes, new_token=new_token)
        if not token:
            return None

    if oidc.use_post:
        if 'json' in ckwargs:
            raise ValueError('Cannot provide json in a post call')

        if 'data' not in ckwargs:
            ckwargs['data'] = {}
        ckwargs['data']['access_token'] = token
    else:
        if 'headers' not in ckwargs:
            ckwargs['headers'] = {}
        ckwargs['headers']['Authorization'] = 'Bearer %s' % token

    resp = requests.request(verb, *args, **ckwargs)
    if resp.status_code == 401 and not is_retry:
        if not auto_refresh:
            return resp

        oidc.token_to_try = oidc.report_token_issue()
        if not oidc.token_to_try:
            return resp
        return _send_oidc_request(oidc, verb, *args, **kwargs)
    elif resp.status_code == 401:
        # We got a 401 and this is a retry. Report error
        oidc.report_token_issue()
        return resp
    else:
        return resp

def send_authorized_request(verb, server, id_provider, url, body, **kwargs):
    """
    Sends authorized request to server.
    """
    if not server:
        server = DEFAULT_MBS_SERVER
    if not id_provider:
        id_provider = DEFAULT_ID_PROVIDER

    logging.info("Trying to get the token from %s", id_provider)

    # Get the auth token using the OpenID client.
    oidc = openidc_client.OpenIDCClient(
        "mbs_build", id_provider,
        {'Token': 'Token', 'Authorization': 'Authorization'},
        'mbs-authorizer', "notsecret")

    scopes = ['openid', 'https://id.fedoraproject.org/scope/groups',
              'https://mbs.fedoraproject.org/oidc/submit-build']

    logging.debug("Sending body: %s", body)
    resp = _send_oidc_request(oidc, verb, urljoin(server, url), json=body,
                              scopes=scopes, **kwargs)
    return resp

def get_scm_url(scm_url, pyrpkg, local=False):
    """
    If `scm_url` it not set, returns the scm_url based on git repository
    in the `os.getcwd()`. When local is True, file:// scheme is used,
    otherwise `pyrpkg` is used to determine public URL to git repository.
    """
    if scm_url:
        return scm_url

    logging.info("You have not provided SCM URL or branch. Trying to get "
        "it from current working directory")

    if local:
        # Just get the local URL from the current working directory.
        scm_url = "file://%s" % os.getcwdu()
        return scm_url
    else:
        # Get the url using pyrpkg implementation.
        process = subprocess.Popen([pyrpkg, 'giturl'], stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
        out, err = process.communicate()
        if process.returncode != 0 and len(err) != 0:
            logging.error("Cannot get the giturl from current "
                "working directory using the %s", pyrpkg)
            logging.error(err)
            return None
        scm_url = out[:-1] # remove new-line
        return scm_url

def get_scm_branch(branch):
    """
    If `branch` it not set, returns the branch name based on git repository
    in the `os.getcwd()`.
    """
    if branch:
        return branch

    process = subprocess.Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = process.communicate()
    if process.returncode != 0 and len(err) != 0:
        logging.error("Cannot get the branch name from current "
            "working directory.")
        logging.error(err)
        return None
    branch = out[:-1] # remove new-line
    return branch

def submit_module_build(scm_url, branch, server, id_provider, pyrpkg, verify=True):
    """
    Submits the module defined by `scm_url` to MBS instance defined
    by `server`. Returns build_id or negative error code.
    """
    scm_url = get_scm_url(scm_url, pyrpkg)
    branch = get_scm_branch(branch)
    if not scm_url or not branch:
        return -2

    logging.info("Submitting module build %s", scm_url)
    body = {'scmurl': scm_url, 'branch': branch}
    resp = send_authorized_request(
        "POST", server, id_provider, "/module-build-service/1/module-builds/",
        body, verify=verify)
    logging.info(resp.text)

    data = resp.json()
    if 'id' in data:
        return data['id']
    return -3

def do_local_build(scm_url, branch):
    """
    Starts the local build using the 'mbs-manager build_module_locally'
    command. Returns exit code of that command or None when scm_url or
    branch are not set and cannot be obtained from the CWD.
    """
    scm_url = get_scm_url(scm_url, None, local=True)
    branch = get_scm_branch(branch)
    if not scm_url or not branch:
        return None

    logging.info("Starting local build of %s, branch %s", scm_url, branch)
    process = subprocess.Popen(['mbs-manager', 'build_module_locally',
                                scm_url, branch])
    process.communicate()
    return process.returncode

def cancel_module_build(server, id_provider, build_id, verify=True):
    """
    Cancels the module build.
    """
    logging.info("Cancelling module build %s", build_id)
    resp = send_authorized_request(
        "PATCH", server, id_provider,
        "/module-build-service/1/module-builds/" + str(build_id),
        {'state': 'failed'}, verify=verify)
    logging.info(resp.text)

def show_overview(server, finished, limit=30):
    if not server:
        server = DEFAULT_MBS_SERVER

    # Base URL to query.
    baseurl = server + '/module-build-service/1/module-builds/'

    # This logging would break our formatting.
    logging.getLogger("requests").setLevel(logging.WARNING)
    logging.getLogger("urllib3").setLevel(logging.WARNING)

    def get_module_builds(page=1, state=0):
        """
        Yields modules with state `state`.
        """
        response = requests.get(baseurl, params=dict(page=page, state=state))
        data = response.json()
        for item in data['items']:
            yield item
        if data['meta']['pages'] > page:
            for item in get_module_builds(page=page+1, state=state):
                yield item

    def get_module_info(module):
        """
        Returns the row with module_info.
        """
        idx = module['id']
        response = requests.get(baseurl + '/%i?verbose=true' % idx)
        module = response.json()
        n_components = len(module['tasks'].get('rpms', []))
        n_built_components = len([c for c in module['tasks'].get('rpms', {}).values() if c['state'] not in [None, 0, koji.BUILD_STATES["BUILDING"]]])
        row = [module["id"], module["state_name"], module["time_submitted"],
               "%s/%s" % (n_built_components, n_components), module["owner"],
               "%s-%s-%s" % (module["name"], module["stream"], module["version"])]
        return row

    if finished:
        # these are the states when the module build is finished
        states = [BUILD_STATES["done"], BUILD_STATES["ready"],
                  BUILD_STATES["failed"]]
    else:
        # this is when the build is in progress
        states = [BUILD_STATES["init"], BUILD_STATES["wait"],
                  BUILD_STATES["build"]]

    # Get all modules in the states we are interested in using 3 threads.
    pool = ThreadPool(3)
    module_builds = pool.map(lambda x: list(get_module_builds(state=x)),
                             states)
    # Make one flat list with all the modules.
    module_builds = [item for sublist in module_builds for item in sublist]
    module_builds.sort(key=lambda x: x["id"])

    # Get the table rows with information about each module using 20 threads.
    pool = ThreadPool(20)
    # get most recent builds
    table = pool.map(get_module_info, module_builds[-limit:])

    # Sort it according to 'id' (first element in list).
    table = list(reversed(sorted(
        table, key=operator.itemgetter(0),
    )))

    # Headers for table we will show to user.
    headers = ["ID", "State", "Submitted", "Components", "Owner", "Module"]

    print(tabulate(table, headers=headers))

def main():
    # Parse command line arguments
    parser = argparse.ArgumentParser(description="Submits and manages module builds.")
    subparsers = parser.add_subparsers(dest="cmd_name")
    parser.add_argument('-v', dest='verbose', action='store_true',
                        help="shows verbose output")
    parser.add_argument('-q', dest='quiet', action='store_true',
                        help="shows only warnings and errors")
    parser.add_argument('-k', '--insecure', dest='verify', action='store_false',
                        help="allow connections to SSL sites without certs")
    parser.add_argument('-s', dest='server', action='store',
                        help="defines the hostname[:port] of the Module Build Service")
    parser.add_argument('-i', dest='idprovider', action='store',
                        help="defines the OpenID Connect identity provider")
    parser.add_argument('-p', dest='pyrpkg_client', action='store',
                        help="defines the name of pyrpkg client executable",
                        default="fedpkg")

    parser_submit = subparsers.add_parser(
        'submit', help="submit module build",
        description="Submits the module build. When 'scm_url' or 'branch' "
        "is not set, it presumes you are executing this command in "
        "the directory with the cloned git repository with a module.")
    parser_submit.add_argument("scm_url", nargs='?')
    parser_submit.add_argument("branch", nargs='?')
    parser_submit.add_argument('-w', dest="watch", action='store_true',
                               help="watch the build progress")

    parser_watch = subparsers.add_parser(
        'watch', help="watch module build",
        description="Watches the build progress of a build submitted by "
        "the 'submit' subcommand.")
    parser_watch.add_argument("build_id")

    parser_info = subparsers.add_parser(
        'info', help="display detailed information about selected module build",
        description="Display detailed information about selected module build.")
    parser_info.add_argument("build_id")

    parser_cancel = subparsers.add_parser(
        'cancel', help="cancel module build",
        description="Cancels the build submitted by 'submit' subcommand.")
    parser_cancel.add_argument("build_id")

    parser_local = subparsers.add_parser(
        'local', help="do local build of module",
        description="Starts local build of a module using the Mock backend. "
        "When 'scm_url' or 'branch' is not set, it presumes you are "
        "executing this command in the directory with the cloned git "
        "repository with a module.")
    parser_local.add_argument("scm_url", nargs='?')
    parser_local.add_argument("branch", nargs='?')

    parser_overview = subparsers.add_parser(
        'overview', help="show overview of module builds",
        description="Shows overview of module builds.")
    parser_overview.add_argument(
        '--finished', dest='finished', action='store_true', default=False,
        help="show only finished module builds")
    parser_overview.add_argument(
        '--limit', dest='limit', action='store', type=int, default=30,
        help="the number of recent builds to show")

    args = parser.parse_args()

    # Initialize the logging.
    if args.verbose:
        loglevel = logging.DEBUG
    elif args.quiet:
        loglevel = logging.WARNING
    else:
        loglevel = logging.INFO
    logging.basicConfig(level=loglevel, format="%(levelname)s: %(message)s")

    if args.cmd_name == "submit":
        # Submit the module build.
        build_id = submit_module_build(args.scm_url, args.branch, args.server,
                                    args.idprovider, args.pyrpkg_client, args.verify)
        if build_id < 0:
            sys.exit(build_id)

        if args.watch:
            watch_build(args.server, build_id)
        else:
            logging.info("Submitted module build %r" % build_id)
    elif args.cmd_name == "local":
        sys.exit(do_local_build(args.scm_url, args.branch))
    elif args.cmd_name == "watch":
        # Watch the module build.
        try:
            watch_build(args.server, args.build_id)
        except KeyboardInterrupt:
            pass
    elif args.cmd_name == "cancel":
        # Cancel the module build
        cancel_module_build(args.server, args.idprovider, args.build_id, args.verify)
    elif args.cmd_name == "overview":
        show_overview(args.server, finished=args.finished, limit=args.limit)
    elif args.cmd_name == "info":
        show_module_info(args.server, args.build_id)

if __name__ == "__main__":
    main()