Remove deprecated mbs-build tool

This commit is contained in:
mprahl
2018-04-03 10:09:29 -04:00
parent 0defabc88c
commit a4903876f2
4 changed files with 0 additions and 610 deletions

View File

@@ -58,21 +58,6 @@ altering your default Copr configuration.
_`Client tooling`
=================
``mbs-build``
-------------
This command submits and manages module builds.
The most frequently used subcommand would be 'submit'. After providing
access credentials, a module build is passed to a preconfigured
MBS instanace. When 'scm_url' or 'branch' is not set, it presumes you
are executing this command in a directory with a cloned git repository
containing a module prescript. The same approach is used in the case of
local module submission.
Other subcommands allow local module submission, watching module builds,
canceling them etc. For more info, there's an existing help available.
``mbs-manager``
---------------

View File

@@ -1,577 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os
import sys
import openidc_client
import argparse
import logging
import subprocess
import requests
import koji
import time
import operator
from tabulate import tabulate
from multiprocessing.dummy import Pool as ThreadPool
from copy import copy
import urllib3
import json
import requests_kerberos
try:
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
DEFAULT_ID_PROVIDER = "https://id.fedoraproject.org/openidc/"
DEFAULT_MBS_SERVER = "https://mbs.fedoraproject.org"
DEFAULT_MBS_REST_PREFIX = "/module-build-service/1/"
DEFAULT_MBS_REST_API = "{0}module-builds/".format(DEFAULT_MBS_REST_PREFIX)
DEFAULT_KOJI_TASK_URL = "https://koji.fedoraproject.org/koji/taskinfo"
openidc_client.WEB_PORTS = [13747]
BUILD_STATES = {
"init": 0,
"wait": 1,
"build": 2,
"done": 3,
"failed": 4,
"ready": 5,
}
INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()}
def get_auth_method(server, verify=True):
config_url = '{0}{1}about/'.format(server.rstrip('/'), DEFAULT_MBS_REST_PREFIX)
rv = requests.get(config_url, timeout=30, verify=verify)
# Assume that if the connection fails, it's because the config API doesn't
# exist on the server yet
if not rv.ok:
return 'oidc'
rv_json = rv.json()
return rv_json['auth_method']
def fetch_module_info(server, build_id):
if not server:
server = DEFAULT_MBS_SERVER
idx = int(build_id)
response = requests.get(server + '%s/%i?verbose=true' % (DEFAULT_MBS_REST_API, idx))
return response.json()
def show_module_info(server, build_id):
state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()])
state_names[None] = "undefined"
data = fetch_module_info(server, build_id)
table = []
for package_name, task_data in data["tasks"].get("rpms", {}).items():
try:
koji_task_url = "%s?taskID=%s" % (DEFAULT_KOJI_TASK_URL, task_data['task_id'])
except KeyError:
koji_task_url = ""
table += [[
task_data.get("nvr", "null"),
state_names[task_data.get("state", None)],
koji_task_url
]]
headers = ["NVR", "State", "Koji Task"]
print(tabulate(table, headers=headers))
def watch_build(server, build_id):
"""
Watches the MBS build in a loop, updates every 30 seconds.
Returns when build state is 'failed' or 'done' or 'ready' or when
user hits ctrl+c.
"""
done = False
while not done:
# Clear the screen
print(chr(27) + "[2J")
state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()])
state_names[None] = "undefined"
data = fetch_module_info(server, build_id)
tasks = dict()
if 'rpms' in data['tasks']:
tasks = data['tasks']['rpms']
states = list(set([task['state'] for task in tasks.values()]))
inverted = dict()
for name, task in tasks.items():
state = task['state']
inverted[state] = inverted.get(state, [])
inverted[state].append(name)
if 0 in inverted:
print("Still building:")
for name in inverted[0]:
task = tasks[name]
print(" ", name, "%s?taskID=%s" % (DEFAULT_KOJI_TASK_URL, task['task_id']))
if 3 in inverted:
print("Failed:")
for name in inverted[3]:
task = tasks[name]
print(" ", name, "%s?taskID=%s" % (DEFAULT_KOJI_TASK_URL, task['task_id']))
print()
print("Summary:")
for state in states:
print(" ", len(inverted[state]), "components in the", state_names[state], "state")
done = data["state_name"] in ["failed", "done", "ready"]
template = ('{owner}\'s build #{id} of {name}-{stream} is in '
'the "{state_name}" state')
if data['state_reason']:
template += ' (reason: {state_reason})'
if data.get('koji_tag'):
template += ' (koji tag: "{koji_tag}")'
print(template.format(**data))
if not done:
time.sleep(30)
# Ideally we would use oidc.send_request here, but it doesn't support
# custom HTTP verbs/methods like "PATCH". It sends just "POST"...
# TODO: Remove this method once python-openidc-client with verb support
# is released and updated in Fedora.
def _send_oidc_request(oidc, verb, *args, **kwargs):
ckwargs = copy(kwargs)
scopes = ckwargs.pop('scopes')
new_token = ckwargs.pop('new_token', True)
auto_refresh = ckwargs.pop('auto_refresh', True)
is_retry = False
if oidc.token_to_try:
is_retry = True
token = oidc.token_to_try
oidc.token_to_try = None
else:
token = oidc.get_token(scopes, new_token=new_token)
if not token:
return None
if oidc.use_post:
if 'json' in ckwargs:
raise ValueError('Cannot provide json in a post call')
if 'data' not in ckwargs:
ckwargs['data'] = {}
ckwargs['data']['access_token'] = token
else:
if 'headers' not in ckwargs:
ckwargs['headers'] = {}
ckwargs['headers']['Authorization'] = 'Bearer %s' % token
resp = requests.request(verb, *args, **ckwargs)
if resp.status_code == 401 and not is_retry:
if not auto_refresh:
return resp
oidc.token_to_try = oidc.report_token_issue()
if not oidc.token_to_try:
return resp
return _send_oidc_request(oidc, verb, *args, **kwargs)
elif resp.status_code == 401:
# We got a 401 and this is a retry. Report error
oidc.report_token_issue()
return resp
else:
return resp
def send_authorized_request(verb, server, url, body, id_provider=None, **kwargs):
"""
Sends authorized request to server.
"""
if not server:
server = DEFAULT_MBS_SERVER
full_url = urljoin(server, url)
verify = kwargs.get('verify', True)
auth_method = get_auth_method(server, verify=verify)
if auth_method == 'oidc':
if not id_provider:
id_provider = DEFAULT_ID_PROVIDER
logging.info("Trying to get the token from %s", id_provider)
# Get the auth token using the OpenID client.
oidc = openidc_client.OpenIDCClient(
"mbs_build", id_provider,
{'Token': 'Token', 'Authorization': 'Authorization'},
'mbs-authorizer', "notsecret")
scopes = ['openid', 'https://id.fedoraproject.org/scope/groups',
'https://mbs.fedoraproject.org/oidc/submit-build']
logging.debug("Sending body: %s", body)
resp = _send_oidc_request(oidc, verb, full_url, json=body,
scopes=scopes, **kwargs)
elif auth_method == 'kerberos':
if type(body) is dict:
data = json.dumps(body)
else:
data = body
auth = requests_kerberos.HTTPKerberosAuth(mutual_authentication=requests_kerberos.OPTIONAL)
resp = requests.request(verb, full_url, data=data, auth=auth, verify=verify)
if resp.status_code == 401:
logging.error('Authentication using Kerberos failed. Make sure you have a valid '
'Kerberos ticket.')
sys.exit(1)
else:
logging.exception('The MBS server requires an unsupported authentication method of '
'"{0}"'.format(auth_method))
sys.exit(1)
return resp
def get_scm_url(scm_url, pyrpkg, local=False):
"""
If `scm_url` it not set, returns the scm_url based on git repository
in the `os.getcwd()`. When local is True, file:// scheme is used,
otherwise `pyrpkg` is used to determine public URL to git repository.
"""
if scm_url:
return scm_url
logging.info("You have not provided SCM URL or branch. Trying to get "
"it from current working directory")
if local:
# Just get the local URL from the current working directory.
scm_url = "file://%s" % os.getcwdu()
return scm_url
else:
# Get the url using pyrpkg implementation.
process = subprocess.Popen([pyrpkg, 'giturl'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
if process.returncode != 0 and len(err) != 0:
logging.error("Cannot get the giturl from current "
"working directory using the %s", pyrpkg)
logging.error(err)
return None
scm_url = out[:-1] # remove new-line
return scm_url
def get_scm_branch(branch=None):
"""
If `branch` it not set, returns the branch name based on git repository
in the `os.getcwd()`.
"""
if branch:
return branch
process = subprocess.Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
if process.returncode != 0 and len(err) != 0:
logging.error("Cannot get the branch name from current "
"working directory.")
logging.error(err)
return None
branch = out[:-1] # remove new-line
return branch
def submit_module_build(scm_url, branch, server, id_provider, pyrpkg, verify=True, optional=None):
"""
Submits the module defined by `scm_url` to MBS instance defined
by `server`. Returns tuple: build_id or negative error code, error message.
"""
scm_url = get_scm_url(scm_url, pyrpkg)
branch = get_scm_branch(branch)
if not scm_url or not branch:
return -2, None
logging.info("Submitting module build %s", scm_url)
body = {'scmurl': scm_url, 'branch': branch}
optional = optional if optional else []
try:
optional_dict = {y[0]: y[1] for y in [x.split("=", 1) for x in optional]}
except IndexError:
return -5, "Optional arguments are not in a proper arg=value format."
body.update(optional_dict)
resp = send_authorized_request(
"POST", server, DEFAULT_MBS_REST_API, body, id_provider=id_provider, verify=verify)
logging.info(resp.text)
data = resp.json()
if 'error' in data:
return -4, "%s %s: %s" % (data['status'], data['error'], data['message'])
elif 'id' in data:
return data['id'], None
return -3, None
def do_local_build(local_builds_nsvs, log_flag=None, yaml_file=None, stream=None, skiptests=False):
"""
Starts the local build using `mbs-manager build_module_locally`.
"""
command = ['mbs-manager']
command.append('build_module_locally')
if local_builds_nsvs:
for build_id in local_builds_nsvs:
command += ['--add-local-build', build_id]
if not yaml_file:
module_dir = os.getcwd()
module_name = os.path.basename(module_dir)
yaml_file = os.path.join(module_dir, module_name + ".yaml")
command.extend(["--file", yaml_file])
if not stream:
stream = get_scm_branch()
command.extend(["--stream", stream])
if log_flag:
command.append(log_flag)
if skiptests:
command.append("--skiptests")
# Some last minute sanity checks before passing off to the other command.
if not os.path.exists(yaml_file):
logging.error("%s does not exist. Specify --file or check pwd.", yaml_file)
return 1
if not stream:
logging.error("Unable to determine stream. Either execute "
"from a git checkout or pass --stream.")
return 1
logging.info("Starting local build of %s, stream %s", yaml_file, stream)
process = subprocess.Popen(command)
process.communicate()
return process.returncode
def cancel_module_build(server, id_provider, build_id, verify=True):
"""
Cancels the module build.
"""
logging.info("Cancelling module build %s", build_id)
resp = send_authorized_request(
"PATCH", server, "%s/%s" % (DEFAULT_MBS_REST_API, build_id),
{'state': 'failed'}, id_provider=id_provider, verify=verify)
logging.info(resp.text)
def show_overview(server, finished, limit=30, verify=True):
if not server:
server = DEFAULT_MBS_SERVER
# Base URL to query.
baseurl = server + DEFAULT_MBS_REST_API
# This logging would break our formatting.
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
def get_module_builds(page=1, state=0):
"""
Yields modules with state `state`.
"""
response = requests.get(baseurl, params=dict(page=page, state=state), verify=verify)
data = response.json()
for item in data['items']:
yield item
if data['meta']['pages'] > page:
for item in get_module_builds(page=page+1, state=state):
yield item
def get_module_info(module):
"""
Returns the row with module_info.
"""
idx = module['id']
response = requests.get(baseurl + '/%i?verbose=true' % idx)
module = response.json()
n_components = len(module['tasks'].get('rpms', []))
n_built_components = len([c for c in module['tasks'].get('rpms', {}).values() if c['state'] not in [None, 0, koji.BUILD_STATES["BUILDING"]]])
row = [module["id"], module["state_name"], module["time_submitted"],
"%s/%s" % (n_built_components, n_components), module["owner"],
"%s-%s-%s" % (module["name"], module["stream"], module["version"])]
return row
if finished:
# these are the states when the module build is finished
states = [BUILD_STATES["done"], BUILD_STATES["ready"],
BUILD_STATES["failed"]]
else:
# this is when the build is in progress
states = [BUILD_STATES["init"], BUILD_STATES["wait"],
BUILD_STATES["build"]]
# Get all modules in the states we are interested in using 3 threads.
pool = ThreadPool(3)
module_builds = pool.map(lambda x: list(get_module_builds(state=x)),
states)
# Make one flat list with all the modules.
module_builds = [item for sublist in module_builds for item in sublist]
module_builds.sort(key=lambda x: x["id"])
# Get the table rows with information about each module using 20 threads.
pool = ThreadPool(20)
# get most recent builds
table = pool.map(get_module_info, module_builds[-limit:])
# Sort it according to 'id' (first element in list).
table = list(reversed(sorted(
table, key=operator.itemgetter(0),
)))
# Headers for table we will show to user.
headers = ["ID", "State", "Submitted", "Components", "Owner", "Module"]
print(tabulate(table, headers=headers))
def main():
print('mbs-build is deprecated in favor of fedpkg and rhpkg. Please transition to '
'using those.', file=sys.stderr)
# Parse command line arguments
parser = argparse.ArgumentParser(description="Submits and manages module builds.")
subparsers = parser.add_subparsers(dest="cmd_name")
# logging
flag_debug = '-d'
flag_verbose = '-v'
flag_quiet = '-q'
parser.add_argument(flag_debug, dest='debug', action='store_true',
help="shows debug output")
parser.add_argument(flag_verbose, dest='verbose', action='store_true',
help="shows verbose output")
parser.add_argument(flag_quiet, dest='quiet', action='store_true',
help="shows only errors")
parser.add_argument('-k', '--insecure', dest='verify', action='store_false',
help="allow connections to SSL sites without certs")
parser.add_argument('-s', dest='server', action='store',
help="defines the hostname[:port] of the Module Build Service")
parser.add_argument('-i', dest='idprovider', action='store',
help="defines the OpenID Connect identity provider")
parser.add_argument('-p', dest='pyrpkg_client', action='store',
help="defines the name of pyrpkg client executable",
default="fedpkg")
parser_submit = subparsers.add_parser(
'submit', help="submit module build",
description="Submits the module build. When 'scm_url' or 'branch' "
"is not set, it presumes you are executing this command in "
"the directory with the cloned git repository with a module.")
parser_submit.add_argument("scm_url", nargs='?')
parser_submit.add_argument("branch", nargs='?')
parser_submit.add_argument('-o', dest="optional", action='append',
help="optional arguments in arg=value format")
parser_submit.add_argument('-w', dest="watch", action='store_true',
help="watch the build progress")
parser_watch = subparsers.add_parser(
'watch', help="watch module build",
description="Watches the build progress of a build submitted by "
"the 'submit' subcommand.")
parser_watch.add_argument("build_id")
parser_info = subparsers.add_parser(
'info', help="display detailed information about selected module build",
description="Display detailed information about selected module build.")
parser_info.add_argument("build_id")
parser_cancel = subparsers.add_parser(
'cancel', help="cancel module build",
description="Cancels the build submitted by 'submit' subcommand.")
parser_cancel.add_argument("build_id")
parser_local = subparsers.add_parser(
'local', help="do local build of module",
description="Starts local build of a module using the Mock backend. "
"When --file or --stream are not set, it presumes you are "
"executing this command in the directory with the cloned git "
"repository with a modulemd yaml file present.")
parser_local.add_argument("--add-local-build", "-l", action='append',
dest="local_builds_nsvs", metavar='BUILD_ID')
parser_local.add_argument('--file', dest='file', action='store',
help="Path to the modulemd yaml file")
parser_local.add_argument('--stream', dest='stream', action='store',
help=("Name of the stream of this build."
" (builds from files only)"))
parser_local.add_argument('--skiptests', dest='skiptests', action='store_true',
help="Path to the modulemd yaml file")
parser_overview = subparsers.add_parser(
'overview', help="show overview of module builds",
description="Shows overview of module builds.")
parser_overview.add_argument(
'--finished', dest='finished', action='store_true', default=False,
help="show only finished module builds")
parser_overview.add_argument(
'--limit', dest='limit', action='store', type=int, default=30,
help="the number of recent builds to show")
args = parser.parse_args()
# Initialize the logging.
log_flag = None
if args.debug:
loglevel = logging.DEBUG
log_flag = flag_debug
elif args.verbose:
loglevel = logging.INFO
log_flag = flag_verbose
elif args.quiet:
loglevel = logging.ERROR
log_flag = flag_quiet
else:
loglevel = logging.WARNING
logging.basicConfig(level=loglevel, format="%(levelname)s: %(message)s")
if args.verify is False:
urllib3.disable_warnings()
if args.cmd_name == "submit":
# Submit the module build.
build_id, errmsg = submit_module_build(args.scm_url, args.branch, args.server,
args.idprovider, args.pyrpkg_client, args.verify, args.optional)
if build_id < 0:
if errmsg:
logging.critical(errmsg)
sys.exit(build_id)
if args.watch:
watch_build(args.server, build_id)
else:
print("Submitted module build %r" % build_id)
elif args.cmd_name == "local":
sys.exit(do_local_build(args.local_builds_nsvs, log_flag, args.file, args.stream,
args.skiptests))
elif args.cmd_name == "watch":
# Watch the module build.
try:
watch_build(args.server, args.build_id)
except KeyboardInterrupt:
pass
elif args.cmd_name == "cancel":
# Cancel the module build
cancel_module_build(args.server, args.idprovider, args.build_id, args.verify)
elif args.cmd_name == "overview":
show_overview(args.server, finished=args.finished, limit=args.limit, verify=args.verify)
elif args.cmd_name == "info":
show_module_info(args.server, args.build_id)
if __name__ == "__main__":
main()

View File

@@ -90,23 +90,6 @@ Prior to starting MBS, you can force development mode::
$ export MODULE_BUILD_SERVICE_DEVELOPER_ENV=1
Module Submission
-----------------
You can submit a local test build with the `contrib/mbs-build` script,
which should submit an HTTP POST to the frontend, requesting a build::
$ ./contrib/mbs-build -s [server] submit [scm_url] [branch]
Here, `server` should specify the `hostname[:port]` port of the MBS instance
you want to submit to. For local development, try `https://127.0.0.1:5000`.
The `scmurl` should be a url to a dist-git repo of the module in question and
the `branch` should be the stream that you want to build. Note that
authentication will be required for submitting a module build. Follow the
on-screen instructions to authenticate.
See also `SCMURLS` in `conf/config.py` for list of allowed SCM URLs.
fedmsg Signing for Development
------------------------------

View File

@@ -62,7 +62,6 @@ setup(name='module-build-service',
'db = module_build_service.resolver.DBResolver:DBResolver',
],
},
scripts=["contrib/mbs-build"],
data_files=[('/etc/module-build-service/', ['conf/cacert.pem',
'conf/config.py',
'conf/copr.conf',